wip

EAGLE3 is an encoder-decoder based speculative decoding method:
- Extracts features from target model at specific layers
- Uses feature fusion layer to compress target features
- Generates draft tokens with single-layer decoder
- Maps draft vocabulary to target vocabulary via d2t tensor

Key changes:
- Add LLM_ARCH_EAGLE3 architecture
- Add EAGLE3 encoder/decoder graph (src/models/eagle3.cpp)
- Add feature extraction from target model layers
- Add g_embeddings handling for decoder input
- Add GGML_TENSOR_FLAG_SYNC for GPU synchronization
- Add --eagle3 flag for speculative-simple example
- Add EAGLE3 model conversion in convert_hf_to_gguf.py

.github/workflows/gguf-publish.yml

@@ -29,10 +29,10 @@ jobs:
       uses: actions/setup-python@v6
       with:
         python-version: '3.11'
-        pip-install: poetry==2.4.0
     - name: Install dependencies
       run: |
         cd gguf-py
+        python -m pip install poetry==2.3.2
         poetry install
 
     - name: Build package

.gitignore

@@ -105,8 +105,6 @@
 __pycache__/
 */poetry.lock
 poetry.toml
-poetry.lock
-uv.lock
 
 # Nix
 

CODEOWNERS

@@ -76,7 +76,6 @@
 /ggml/src/ggml-vulkan/                  @ggml-org/ggml-vulkan
 /ggml/src/ggml-webgpu/                  @ggml-org/ggml-webgpu
 /ggml/src/ggml-zdnn/                    @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
-/ggml/src/ggml-zendnn/                  @avinashcpandey @Jiten1parmar @z-vishal
 /ggml/src/ggml.c                        @ggerganov
 /ggml/src/ggml.cpp                      @ggerganov
 /ggml/src/gguf.cpp                      @JohannesGaessler @Green-Sky

README.md

@@ -529,7 +529,6 @@ To learn more about model quantization, [read this documentation](tools/quantize
 - [How to build](docs/build.md)
 - [Running on Docker](docs/docker.md)
 - [Build on Android](docs/android.md)
-- [Multi-GPU usage](docs/multi-gpu.md)
 - [Performance troubleshooting](docs/development/token_generation_performance_tips.md)
 - [GGML tips & tricks](https://github.com/ggml-org/llama.cpp/wiki/GGML-Tips-&-Tricks)
 

common/arg.cpp

@@ -248,8 +248,6 @@ std::vector<std::string> common_arg::get_env() const {
 
 // Helper function to parse tensor buffer override strings
 static void parse_tensor_buffer_overrides(const std::string & value, std::vector<llama_model_tensor_buft_override> & overrides) {
-    ggml_backend_load_all();
-
     std::map<std::string, ggml_backend_buffer_type_t> buft_list;
     for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
         auto * dev = ggml_backend_dev_get(i);
@@ -427,10 +425,6 @@ static bool parse_bool_value(const std::string & value) {
     }
 }
 
-[[noreturn]] static void arg_removed(const std::string & msg) {
-    throw std::invalid_argument("the argument has been removed. " + msg);
-}
-
 //
 // CLI argument parsing functions
 //
@@ -809,7 +803,6 @@ static std::vector<ggml_backend_dev_t> parse_device_list(const std::string & val
     if (dev_names.size() == 1 && dev_names[0] == "none") {
         devices.push_back(nullptr);
     } else {
-        ggml_backend_load_all();
         for (const auto & device : dev_names) {
             auto * dev = ggml_backend_dev_by_name(device.c_str());
             if (!dev || ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
@@ -827,7 +820,6 @@ static void add_rpc_devices(const std::string & servers) {
     if (rpc_servers.empty()) {
         throw std::invalid_argument("no RPC servers specified");
     }
-    ggml_backend_load_all();
     ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC");
     if (!rpc_reg) {
         throw std::invalid_argument("failed to find RPC backend");
@@ -1024,6 +1016,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
 
     params.use_color = tty_can_use_colors();
 
+    // load dynamic backends
+    ggml_backend_load_all();
+
     common_params_context ctx_arg(params);
     ctx_arg.print_usage = print_usage;
     ctx_arg.ex          = ex;
@@ -2280,7 +2275,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         {"--list-devices"},
         "print list of available devices and exit",
         [](common_params &) {
-            ggml_backend_load_all();
             std::vector<ggml_backend_dev_t> devices;
             for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
                 auto * dev = ggml_backend_dev_get(i);
@@ -3073,7 +3067,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params, bool value) {
             params.use_jinja = value;
         }
-    ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_MTMD}).set_env("LLAMA_ARG_JINJA"));
+    ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SPECULATIVE}).set_env("LLAMA_ARG_JINJA"));
     add_opt(common_arg(
         {"--reasoning-format"}, "FORMAT",
         "controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:\n"
@@ -3129,7 +3123,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params, const std::string & value) {
             params.chat_template = value;
         }
-    ).set_examples({LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MTMD}).set_env("LLAMA_ARG_CHAT_TEMPLATE"));
+    ).set_examples({LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SPECULATIVE}).set_env("LLAMA_ARG_CHAT_TEMPLATE"));
     add_opt(common_arg(
         {"--chat-template-file"}, "JINJA_TEMPLATE_FILE",
         string_format(
@@ -3461,6 +3455,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.speculative.draft.cache_type_v = kv_cache_type_from_str(value);
         }
     ).set_env("LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V"));
+    // TODO: rename:
+    add_opt(common_arg(
+        {"--eagle3"},
+        "use EAGLE3 speculative decoding with the draft model",
+        [](common_params & params) {
+            params.speculative.draft.eagle3 = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_CLI}));
     add_opt(common_arg(
         {"--spec-draft-override-tensor", "-otd", "--override-tensor-draft"}, "<tensor name pattern>=<buffer type>,...",
         "override tensor buffer type for draft model", [](common_params & params, const std::string & value) {
@@ -3721,35 +3723,35 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         {"--draft", "--draft-n", "--draft-max"}, "N",
         "the argument has been removed. use --spec-draft-n-max or --spec-ngram-mod-n-max",
         [](common_params & /*params*/, int /*value*/) {
-            arg_removed("use --spec-draft-n-max or --spec-ngram-mod-n-max");
+            throw std::invalid_argument("the argument has been removed. use --spec-draft-n-max or --spec-ngram-mod-n-max");
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DRAFT_MAX"));
     add_opt(common_arg(
         {"--draft-min", "--draft-n-min"}, "N",
         "the argument has been removed. use --spec-draft-n-min or --spec-ngram-mod-n-min",
         [](common_params & /*params*/, int /*value*/) {
-            arg_removed("use --spec-draft-n-min or --spec-ngram-mod-n-min");
+            throw std::invalid_argument("the argument has been removed. use --spec-draft-n-min or --spec-ngram-mod-n-min");
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DRAFT_MIN"));
     add_opt(common_arg(
         {"--spec-ngram-size-n"}, "N",
         "the argument has been removed. use the respective --spec-ngram-*-size-n or --spec-ngram-mod-n-match",
         [](common_params & /*params*/, int /*value*/) {
-            arg_removed("use the respective --spec-ngram-*-size-n");
+            throw std::invalid_argument("the argument has been removed. use the respective --spec-ngram-*-size-n");
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"--spec-ngram-size-m"}, "N",
         "the argument has been removed. use the respective --spec-ngram-*-size-m",
         [](common_params & /*params*/, int /*value*/) {
-            arg_removed("use the respective --spec-ngram-*-size-m");
+            throw std::invalid_argument("the argument has been removed. use the respective --spec-ngram-*-size-m");
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"--spec-ngram-min-hits"}, "N",
         "the argument has been removed. use the respective --spec-ngram-*-min-hits",
         [](common_params & /*params*/, int /*value*/) {
-            arg_removed("use the respective --spec-ngram-*-min-hits");
+            throw std::invalid_argument("the argument has been removed. use the respective --spec-ngram-*-min-hits");
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER}));
 

common/chat-auto-parser-generator.cpp

@@ -369,7 +369,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
                                            arguments.name_suffix) +
                            arguments.value_prefix +
                            (schema_info.resolves_to_string(param_schema) ?
-                                p.tool_arg_string_value(until_suffix) :
+                                p.tool_arg_string_value(p.schema(until_suffix,
+                                                                 "tool-" + name + "-arg-" + param_name + "-schema",
+                                                                 param_schema, true)) :
                                 p.tool_arg_json_value(p.schema(
                                     p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +
                                     p.space()) +

common/chat.cpp

@@ -80,7 +80,7 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
     if (!content.empty()) {
         jmsg["content"] = content;
     } else if (!content_parts.empty()) {
-        if (concat_typed_text || contains_media()) {
+        if (concat_typed_text) {
             std::string text;
             bool last_was_media_marker = false;
             // join parts with newline, do not add newline before or after media markers

common/chat.h

@@ -94,15 +94,6 @@ struct common_chat_msg {
                tool_name.empty() && tool_call_id.empty();
     }
 
-    bool contains_media() const {
-        for (const auto & part : content_parts) {
-            if (part.type == "media_marker") {
-                return true;
-            }
-        }
-        return false;
-    }
-
     void set_tool_call_ids(std::vector<std::string> &           ids_cache,
                            const std::function<std::string()> & gen_tool_call_id) {
         for (auto i = 0u; i < tool_calls.size(); i++) {

common/common.h

@@ -307,10 +307,13 @@ struct common_params_speculative_draft {
 
     common_params_model mparams;
 
-    llama_model * model = nullptr; // a llama_model that can be shared by multiple speculative contexts
+    llama_model * model     = nullptr; // a llama_model that can be shared by multiple speculative contexts
+    llama_model * model_tgt = nullptr; // the target model
 
     llama_context_params cparams; // these are the parameters for the draft llama_context
 
+    bool    eagle3       = false; // use EAGLE3 speculative decoding
+
     int32_t n_ctx        = 0;  // draft context size
     int32_t n_gpu_layers = -1; // number of layers to store in VRAM for the draft model (-1 - use default)
 

common/fit.cpp

@@ -109,24 +109,16 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data(
         ret.back().total = total;
     }
     for (size_t i = 0; i < nd; i++) {
-        ggml_backend_dev_t dev = llama_model_get_device(model, i);
-
         size_t free;
         size_t total;
-        ggml_backend_dev_memory(dev, &free, &total);
+        ggml_backend_dev_memory(llama_model_get_device(model, i), &free, &total);
 
-        // Some non-GPU accelerator backends, such as BLAS, report 0/0 and rely on
-        // the host-memory fallback. For GPU-like backends, keep 0/0 so --fit does
-        // not assign anything to a device with an unknown memory budget.
+        // devices can return 0 bytes for free and total memory if they do not
+        // have any to report. in this case, we will use the host memory as a fallback
+        // fixes: https://github.com/ggml-org/llama.cpp/issues/18577
         if (free == 0 && total == 0) {
-            const enum ggml_backend_dev_type type = ggml_backend_dev_type(dev);
-            if (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
-                LOG_WRN("%s: device %s did not report memory; --fit will not use it\n",
-                        __func__, ggml_backend_dev_name(dev));
-            } else {
-                free  = ret.back().free;
-                total = ret.back().total;
-            }
+            free  = ret.back().free;
+            total = ret.back().total;
         }
         ret[i].free  = free;
         ret[i].total = total;

common/reasoning-budget.cpp

@@ -158,6 +158,8 @@ static void common_reasoning_budget_apply(struct llama_sampler * smpl, llama_tok
     for (size_t i = 0; i < cur_p->size; i++) {
         if (cur_p->data[i].id != forced) {
             cur_p->data[i].logit = -INFINITY;
+        } else {
+            cur_p->data[i].logit = +INFINITY; // force the token
         }
     }
 }

common/speculative.cpp

@@ -48,6 +48,7 @@ struct common_speculative_config {
             const common_params_speculative & p = common_params_speculative{}) : type(t), params(p) {}
 };
 
+
 static bool common_speculative_are_compatible(
     const llama_model * model_tgt,
     const llama_model * model_dft) {
@@ -240,7 +241,9 @@ struct common_speculative_state_draft : public common_speculative_state {
     ~common_speculative_state_draft() override {
         llama_perf_context_print(ctx_dft);
 
-        llama_free(ctx_dft);
+        if (ctx_dft) {
+            llama_free(ctx_dft);
+        }
 
         common_sampler_free(smpl);
 
@@ -291,11 +294,11 @@ struct common_speculative_state_draft : public common_speculative_state {
 
         auto * spec = this;
 
-        auto & batch      = spec->batch;
-        auto & ctx_tgt    = spec->ctx_tgt;
-        auto & ctx_dft    = spec->ctx_dft;
-        auto & smpl       = spec->smpl;
-        auto & prompt_dft = spec->prompt_dft;
+        auto & batch       = spec->batch;
+        auto & ctx_tgt     = spec->ctx_tgt;
+        auto & ctx_dft     = spec->ctx_dft;
+        auto & smpl        = spec->smpl;
+        auto & prompt_dft  = spec->prompt_dft;
 
         auto * mem_dft = llama_get_memory(ctx_dft);
 
@@ -567,7 +570,52 @@ struct common_speculative_state_draft : public common_speculative_state {
 };
 
 struct common_speculative_state_eagle3 : public common_speculative_state {
-    common_speculative_state_eagle3(enum common_speculative_type type) : common_speculative_state(type) {}
+    llama_context * ctx_tgt;
+
+    common_sampler * smpl;
+
+    llama_batch batch;
+
+    struct llama_context * ctx_dft_enc = nullptr;
+    struct llama_context * ctx_dft_dec = nullptr;
+
+    int32_t eagle3_n_past = 0;  // number of verified positions in decoder KV cache
+
+    common_speculative_state_eagle3(
+            enum common_speculative_type type,
+            llama_context * ctx_tgt,
+            llama_context * ctx_dft_enc,
+            llama_context * ctx_dft_dec)
+        : common_speculative_state(type)
+        , ctx_tgt(ctx_tgt)
+        , ctx_dft_enc(ctx_dft_enc)
+        , ctx_dft_dec(ctx_dft_dec)
+    {
+        batch = llama_batch_init(llama_n_batch(ctx_dft_dec), 0, 1);
+
+        // Initialize sampler for EAGLE3 decoder
+        common_params_sampling params;
+        params.no_perf = false;
+        params.top_k = 10; // set 1 for greedy sampling (argmax) to match vLLM's default behavior but >1 always gets higher acceptance rate for eagle3
+        params.samplers = { COMMON_SAMPLER_TYPE_TOP_K };
+        smpl = common_sampler_init(llama_get_model(ctx_dft_dec), params);
+    }
+
+    ~common_speculative_state_eagle3() override {
+        llama_perf_context_print(ctx_dft_dec);
+
+        if (ctx_dft_dec) {
+            llama_free(ctx_dft_dec);
+        }
+
+        if (ctx_dft_enc) {
+            llama_free(ctx_dft_enc);
+        }
+
+        common_sampler_free(smpl);
+
+        llama_batch_free(batch);
+    }
 
     void begin(const llama_tokens & prompt) override {
         GGML_UNUSED(prompt);
@@ -577,12 +625,97 @@ struct common_speculative_state_eagle3 : public common_speculative_state {
             const common_params_speculative & params,
             const llama_tokens & prompt_tgt,
             llama_token id_last,
-            llama_tokens & draft_tokens) override {
-        // TODO: implement
-        GGML_UNUSED(params);
-        GGML_UNUSED(prompt_tgt);
-        GGML_UNUSED(id_last);
-        GGML_UNUSED(draft_tokens);
+            llama_tokens & result) override {
+        auto * spec = this;
+
+        auto & batch       = spec->batch;
+        auto & ctx_tgt     = spec->ctx_tgt;
+        auto & ctx_dft_enc = spec->ctx_dft_enc;
+        auto & ctx_dft_dec = spec->ctx_dft_dec;
+        auto & smpl        = spec->smpl;
+
+        //result = gen_eagle3_draft(spec, params, prompt_tgt, id_last);
+        const int n_embd = llama_model_n_embd(llama_get_model(ctx_dft_enc));
+        const int n      = (int)prompt_tgt.size();
+        const int n_new  = n - spec->eagle3_n_past;
+
+        GGML_ASSERT(n >= 1 && "prompt_tgt is empty");
+        GGML_ASSERT(n_new >= 1 && "must have at least 1 new token");
+
+        // Clear draft positions from decoder KV cache [n_past, inf)
+        llama_memory_seq_rm(llama_get_memory(ctx_dft_dec), 0, spec->eagle3_n_past, -1);
+
+        // Encoder: features → g_embeddings
+        const float * features = llama_get_eagle3_target_features(ctx_tgt);
+        GGML_ASSERT(features && "no target features");
+
+        llama_batch enc_batch = {
+            /*.n_tokens  =*/ n_new,
+            /*.token     =*/ nullptr,
+            /*.embd      =*/ const_cast<float*>(features),
+            /*.pos       =*/ nullptr,
+            /*.n_seq_id  =*/ nullptr,
+            /*.seq_id    =*/ nullptr,
+            /*.logits    =*/ nullptr,
+        };
+        GGML_ASSERT(llama_encode(ctx_dft_enc, enc_batch) == 0);
+
+        const float * g_embd = llama_get_embeddings(ctx_dft_enc);
+        GGML_ASSERT(g_embd && "encoder output failed");
+
+        // Decoder batch: process new tokens with KV cache reuse
+        llama_set_eagle3_g_embeddings(ctx_dft_dec, g_embd, n_embd, n_new);
+
+        common_batch_clear(batch);
+        for (int i = 0; i < n_new; i++) {
+            const int pos = spec->eagle3_n_past + i;
+            const llama_token tok = (pos < n - 1) ? prompt_tgt[pos + 1] : id_last;
+            common_batch_add(batch, tok, pos, {0}, true);
+        }
+
+        GGML_ASSERT(llama_decode(ctx_dft_dec, batch) == 0);
+
+        spec->eagle3_n_past = n;  // update verified positions
+
+        // Sample draft tokens
+        result.clear();
+        common_sampler_reset(smpl);
+
+        // Sample and check probability (consistent with standard speculative decoding)
+        auto sample_and_check = [&](int idx) -> bool {
+            common_sampler_sample(smpl, ctx_dft_dec, idx);
+
+            const auto * cur_p = common_sampler_get_candidates(smpl, true);
+            const llama_token id = cur_p->data[0].id;
+
+            common_sampler_accept(smpl, id, true);
+            result.push_back(id);
+
+            return cur_p->data[0].p >= params.draft.p_min;
+        };
+
+        // First draft token from batch decode
+        if (!sample_and_check(n_new - 1)) {
+            return;
+        }
+
+        // Autoregressive: use prenorm as g_embd (-1 = last output)
+        const float * prenorm = llama_get_embeddings_ith(ctx_dft_dec, -1);
+
+        for (int i = 1; i < params.draft.n_max; i++) {
+            GGML_ASSERT(prenorm && "prenorm failed");
+            llama_set_eagle3_g_embeddings(ctx_dft_dec, prenorm, n_embd, 1);
+
+            common_batch_clear(batch);
+            common_batch_add(batch, result.back(), n - 1 + i, {0}, true);
+            GGML_ASSERT(llama_decode(ctx_dft_dec, batch) == 0);
+
+            prenorm = llama_get_embeddings_ith(ctx_dft_dec, -1);
+
+            if (!sample_and_check(0)) {
+                break;
+            }
+        }
     }
 
     void accept(uint16_t n_accepted) override {
@@ -975,11 +1108,35 @@ common_speculative * common_speculative_init(
         common_params_speculative & params,
         llama_context             * ctx_tgt) {
     llama_context * ctx_dft = nullptr;
+
+    llama_context * ctx_dft_enc = nullptr;
+    llama_context * ctx_dft_dec = nullptr;
+
     if (params.draft.model) {
-        ctx_dft = llama_init_from_model(params.draft.model, params.draft.cparams);
-        if (ctx_dft == nullptr) {
-            LOG_ERR("%s", "failed to create draft context\n");
-            return nullptr;
+        if (params.draft.eagle3) {
+            llama_context_params params_enc = params.draft.cparams;
+            params_enc.target_model = nullptr;
+            params_enc.embeddings = true;
+            ctx_dft_enc = llama_init_from_model(params.draft.model, params_enc);
+            if (!ctx_dft_enc) {
+                LOG_ERR("failed to create EAGLE3 encoder context\n");
+                return nullptr;
+            }
+
+            llama_context_params params_dec = params.draft.cparams;
+            params_dec.target_model = params.draft.model_tgt;
+            params_dec.embeddings = true;
+            ctx_dft_dec = llama_init_from_model(params.draft.model, params_dec);
+            if (!ctx_dft_dec) {
+                LOG_ERR("failed to create EAGLE3 decoder context\n");
+                return nullptr;
+            }
+        } else {
+            ctx_dft = llama_init_from_model(params.draft.model, params.draft.cparams);
+            if (ctx_dft == nullptr) {
+                LOG_ERR("%s", "failed to create draft context\n");
+                return nullptr;
+            }
         }
     }
 
@@ -987,7 +1144,7 @@ common_speculative * common_speculative_init(
     std::vector<common_speculative_config> configs = {}; // list of speculative configs to try
     {
         bool has_draft = !params.draft.mparams.path.empty();
-        bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3
+        bool has_draft_eagle3 = params.draft.eagle3;
 
         bool has_ngram_cache   = (params.type == COMMON_SPECULATIVE_TYPE_NGRAM_CACHE);
         bool has_ngram_simple  = (params.type == COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE);
@@ -1029,10 +1186,11 @@ common_speculative * common_speculative_init(
             configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params));
         }
         if (has_draft) {
-            configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT, params));
-        }
-        if (has_draft_eagle3) {
-            configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_EAGLE3, params));
+            if (has_draft_eagle3) {
+                configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_EAGLE3, params));
+            } else {
+                configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT, params));
+            }
         }
     }
 
@@ -1055,7 +1213,11 @@ common_speculative * common_speculative_init(
                 break;
             }
             case COMMON_SPECULATIVE_TYPE_EAGLE3: {
-                impls.push_back(std::make_unique<common_speculative_state_eagle3>(config.type));
+                impls.push_back(std::make_unique<common_speculative_state_eagle3>(config.type,
+                    /* .ctx_tgt      = */ ctx_tgt,
+                    /* .ctx_dft_enc  = */ ctx_dft_enc,
+                    /* .ctx_dft_dec  = */ ctx_dft_dec
+                ));
                 break;
             }
             case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {

convert_hf_to_gguf_update.py

@@ -175,7 +175,6 @@ pre_computed_hashes = [
     {"name": "falcon-h1", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon-H1-34B-Base", "chkhsh": "48f8e02c0359c0bbdd82f26909171fac1c18a457bb47573ed1fe3bbb2c1cfd4b"},
     {"name": "kimi-k2",   "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/moonshotai/Kimi-K2-Base",   "chkhsh": "81212dc7cdb7e0c1074ca62c5aeab0d43c9f52b8a737be7b12a777c953027890"},
     {"name": "qwen2",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B", "chkhsh": "d4540891389ea895b53b399da6ac824becc30f2fba0e9ddbb98f92e55ca0e97c"},
-    {"name": "qwen35",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openbmb/MiniCPM-V-4_6", "chkhsh": "1444df51289cfa8063b96f0e62b1125440111bc79a52003ea14b6eac7016fd5f"},
     {"name": "grok-2",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"},
     # jina-v2-de variants
     {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},

docs/backend/SYCL.md

@@ -737,14 +737,6 @@ use 1 SYCL GPUs: [0] with Max compute units:512
 | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.<br>Recommended to use when --split-mode = layer |
 | UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Support malloc device memory more than 4GB.|
 
-## Compile-time Flags
-
-Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spot.
-
-| Name            | Function                                                                         |
-|-----------------|----------------------------------------------------------------------------------|
-| DEBUG_SYCL_POOL | Enable device memory pool logging on teardown. Useful for profiling allocations. |
-
 ## Design Rule
 
 - Open to all contributors.

docs/multi-gpu.md

@@ -1,127 +0,0 @@
-# Using multiple GPUs with llama.cpp
-
-This guide explains how to run [llama.cpp](https://github.com/ggml-org/llama.cpp) across more than one GPU. It covers the split modes, the command-line flags that control them, the limitations you need to know about, and ready-to-use recipes for `llama-cli` and `llama-server`.
-
-The CLI arguments listed here are the same for both tools - or most llama.cpp binaries for that matter.
-
----
-
-## When you need multi-GPU
-
-Reach for multi-GPU when one of these is true:
-
-- **The model doesn't fit in a single GPU's VRAM.** By spreading the weights across two or more GPUs the whole model can stay on accelerators. Otherwise part of the model will need to be run off of the comparatively slower system RAM.
-- **You want more throughput.** By distributing the computation across multiple GPUs, each individual GPU has to do less work. This can result in better prefill and/or token generation performance, depending on the split mode and interconnect speed vs. the speed of an individual GPU.
-
----
-
-## The split modes
-
-Set with `--split-mode` / `-sm`.
-
-| Mode | What it does | When to use |
-|---|---|---|
-| `none` | Use a single GPU only. Pick which one with `--main-gpu`. | You explicitly want to confine the model to one GPU even though more are visible. |
-| `layer` (**default**) | Pipeline parallelism. Each GPU holds a contiguous slice of layers. The KV cache for layer *l* lives on the GPU that owns layer *l*. | Default and most compatible multi-GPU choice. You want more memory than a single GPU provides and your priority is a fast prefill. Can tolerate slow interconnect speeds between GPUs. |
-| `row` | **Deprecated.** Older row-split tensor-parallel path with comparatively poor performance. Splits only dense weights across GPUs. Superseded by `tensor` which should be universally superior if it can be used. | Avoid in new deployments. |
-| `tensor` | **EXPERIMENTAL.** Tensor parallelism that splits both weights *and* KV across the participating GPUs via a "meta device" abstraction. | You want more memory than a single GPU provides and your priority is fast token generation. Prefill speeds approach pipeline parallel speeds for large, dense models and fast GPU interconnect speeds. Treat as experimental as the code is less mature than pipeline parallelism. Performance should be good for multiple NVIDIA GPUs using the CUDA backend, no guarantees otherwise. |
-
-> Pipeline parallel (`layer`) vs. tensor parallel (`tensor`): pipeline-parallel runs different layers on different GPUs and processes tokens sequentially through the pipeline. This minimizes data transfers between GPUs but requires many tokens to scale well. Tensor-parallel splits each layer across GPUs and does multiple cross-GPU reductions per layer. This enables parallelizing any workload but is much more bottlenecked by the GPU interconnect speed. Pipeline-parallel maximizes batch throughput; tensor-parallel minimizes latency.
-
----
-
-## Command-line arguments reference
-
-| Short | Long | Value | Default | Notes |
-|---|---|---|---|---|
-| `-sm` | `--split-mode` | `none` \| `layer` \| `tensor` | `layer` | See modes above. |
-| `-ts` | `--tensor-split` | comma-separated proportions, e.g. `3,1` | mode-dependent | How much of the model goes to each GPU. If omitted, `layer`/`row` use automatic splitting proportional to memory, while `tensor` splits tensor segments evenly. With `3,1` on two GPUs, GPU 0 gets 75 %, GPU 1 gets 25 %. The values follow the order in `--device`. |
-| `-mg` | `--main-gpu` | integer device index | `0` | The single GPU used in `--split-mode none`. |
-| `-ngl` | `--n-gpu-layers` / `--gpu-layers` | integer \| `auto` \| `all` | `auto` | Maximum number of layers to keep in VRAM. Use `999` or `all` to push everything possible to the GPUs. |
-| `-dev` | `--device` | comma-separated device names, or `none` | auto | Restrict which devices llama.cpp may use. See `--list-devices` for names. |
-| | `--list-devices` | - | - | Print the available devices and their memory. Run this first to learn the names you'd pass to `--device`. |
-| `-fa` | `--flash-attn` | `on` \| `off` \| `auto` | `auto` | Required when using `--split-mode tensor` and/or quantized V cache. Supported (and therefore enabled by default) for most combinations of models and backends. |
-| `-ctk` | `--cache-type-k` | `f32` \| `f16` \| `bf16` \| `q8_0` \| `q4_0` \| ... | `f16` | KV cache type for K. |
-| `-ctv` | `--cache-type-v` | same as `-ctk` | `f16` | KV cache type for V. |
-| `-fit` | `--fit` | `on` \| `off` | `on` | Auto-fit unset args to device memory. **Not supported with `tensor`. You may need to manually set the `--ctx-size` to make the model fit.**  |
-
-As for any CUDA program, the environment variable `CUDA_VISIBLE_DEVICES` can be used to control which GPUs to use for the CUDA backend: if you set it, llama.cpp only sees the specified GPUs. Use `--device` for selecting GPUs from among those visible to llama.cpp, this works for any backend.
-
----
-
-## Recipes
-
-### 1. Default - pipeline parallel across all visible GPUs
-
-```bash
-llama-cli -m model.gguf
-llama-server -m model.gguf
-```
-
-Easiest configuration. KV cache spreads across the GPUs along with the layers. `--fit` (on by default) sizes things automatically.
-
-### 2. Pipeline parallel with a custom split ratio
-
-```bash
-llama-cli -m model.gguf -ts 3,1
-```
-
-Useful when GPUs have different memory: GPU 0 (3 parts) and GPU 1 (1 part). Proportions are normalized so `-ts 3,1` is the same as e.g. `-ts 75,25`.
-
-### 3. Single-GPU mode, picking a specific GPU
-
-```bash
-llama-cli --list-devices
-llama-cli -m model.gguf -dev CUDA1
-```
-
-Use only the device listed as `CUDA1` when calling with `--list-devices`.
-
-### 4. Tensor parallelism (experimental)
-
-```bash
-llama-cli -m model.gguf -sm tensor -ctk f16 -ctv f16
-```
-
-- `--flash-attn off` or (`--flash-attn auto` resolving to `off` when it isn't supported) is a hard error.
-- KV cache types must be non-quantized: `f32`, `f16`, or `bf16`. Support for quantized KV cache is not implemented and trying to use it will result in an error.
-- Mark this configuration as experimental in your tooling: validate output quality before deploying.
-- `--split-mode tensor`is not implemented for all architectures. The following will fail with *"LLAMA_SPLIT_MODE_TENSOR not implemented for architecture '...'"*:
-
-  - **MoE / hybrid:** Grok, MPT, OLMoE, DeepSeek2, GLM-DSA, Nemotron-H, Nemotron-H-MoE, Granite-Hybrid, LFM2-MoE, Minimax-M2, Mistral4, Kimi-Linear, Jamba, Falcon-H1
-  - **State-space / RWKV-style:** Mamba, Mamba2 (and the hybrid Mamba-attention models above)
-  - **Other:** PLAMO2, MiniCPM3, Gemma-3n, OLMo2, BitNet, T5
-
-### 5. With NCCL
-
-There's no runtime flag for NCCL - it's selected at build time (`-DGGML_CUDA_NCCL=ON`, this is the default). Note that NCCL is **not** automatically distributed with CUDA and you may need to install it manually - when in doubt check the CMake log to see whether or not it can find the package. When llama.cpp is compiled with NCCL support it uses it automatically for cross-GPU reductions in `tensor` mode. When NCCL is missing on a multi-GPU build, you'll see this one-time warning and performance will be lower:
-
-```
-NVIDIA Collective Communications Library (NCCL) is unavailable, multi GPU performance will be suboptimal
-```
-
-When using the "ROCm" backend (which is the ggml CUDA code translated for AMD via HIP), the AMD equivalent RCCL can be used by compiling with `-DGGML_HIP_RCCL=ON`. Note that RCCL is by default *disabled* because (unlike NCCL) it was not universally beneficial during testing.
-### 6. With CUDA peer-to-peer access (`GGML_CUDA_P2P`)
-
-CUDA peer-to-peer (P2P) lets GPUs transfer data directly between each other instead of going through system memory, which generally improves multi-GPU performance. It is **opt-in** at runtime - set the environment variable `GGML_CUDA_P2P` to any value to enable it:
-
-```bash
-GGML_CUDA_P2P=1 llama-cli -m model.gguf -sm tensor
-```
-
-P2P requires driver support (usually restricted to workstation/datacenter GPUs) and **may cause crashes or corrupted outputs on some motherboards or BIOS configurations** (e.g. when IOMMU is enabled). If you see instability after enabling it, unset the variable.
-
----
-
-## Troubleshooting
-
-| Symptom | How to fix |
-|---|---|
-| Startup error *"SPLIT_MODE_TENSOR requires flash_attn to be enabled"* | Add `-fa on` or remove `-fa off`. |
-| Startup error *"simultaneous use of SPLIT_MODE_TENSOR and KV cache quantization not implemented"* | Use `-ctk f16 -ctv f16` (or `bf16`/`f32`) with `--split-mode tensor`. |
-| Startup error *"LLAMA_SPLIT_MODE_TENSOR not implemented for architecture 'X'"* | Architecture not on the TENSOR allow-list. Use `--split-mode layer`. |
-| Warning *"NCCL is unavailable, multi GPU performance will be suboptimal"* | llama.cpp wasn't built with NCCL. Either accept the lower performance or install NCCL and rebuild. |
-| CUDA OOM at startup or during prefill in `--split-mode tensor` | Auto-fit is disabled in this mode, so reduce memory pressure yourself. In order from least to most disruptive: lower `--ctx-size` (`-c`) (KV cache is roughly proportional to `n_ctx`); for `llama-server`, lower `--parallel` (`-np`) (a slot KV cache is allocated per concurrent sequence); as a last resort, reduce `--n-gpu-layers` (`-ngl`) (the remaining layers run on CPU and inference will be much slower). |
-| Performance is worse with multi-GPU than single-GPU | The performance is bottlenecked by GPU interconnect speed. For `--split-mode tensor`, verify that NCCL is being used. Try `--split-mode layer` (less communication than `tensor`). Increase GPU interconnect speed via more PCIe lanes or e.g. NVLink (if available). |
-| GPU not used at all | `--n-gpu-layers` is `0` or too low - try explicitly setting `-ngl all`. Or you are accidentally hiding the GPUs via an environment variable like `CUDA_VISIBLE_DEVICES=-1`. Or your build doesn't include support for the relevant backend. |
-| Crashes or corrupted outputs after setting `GGML_CUDA_P2P=1` | Some motherboards and BIOS settings (e.g. with IOMMU enabled) don't support CUDA peer-to-peer reliably. Unset `GGML_CUDA_P2P`. |

docs/multimodal/minicpmv4.6.md

@@ -1,49 +0,0 @@
-## MiniCPM-V 4.6
-
-### Prepare models and code
-
-Download [MiniCPM-V-4_6](https://huggingface.co/openbmb/MiniCPM-V-4_6) PyTorch model from huggingface to "MiniCPM-V-4_6" folder.
-
-The model must be the standard `transformers` v5.7.0+ checkpoint (no `trust_remote_code`); the architecture in `config.json` is `MiniCPMV4_6ForConditionalGeneration` with a `qwen3_5_text` text model and a SigLIP-based vision tower plus a window-attention `vit_merger`.
-
-### Build llama.cpp
-
-If there are differences in usage, please refer to the official build [documentation](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md)
-
-Clone llama.cpp:
-```bash
-git clone https://github.com/ggml-org/llama.cpp
-cd llama.cpp
-```
-
-Build llama.cpp using `CMake`:
-```bash
-cmake -B build
-cmake --build build --config Release
-```
-
-
-### Usage of MiniCPM-V 4.6
-
-Unlike older MiniCPM-V variants, MiniCPM-V 4.6 is converted directly through `convert_hf_to_gguf.py`. The same script is invoked twice on the original Hugging Face directory: once to produce the language-model GGUF and once with `--mmproj` to produce the multimodal projector GGUF.
-
-```bash
-# language model
-python ./convert_hf_to_gguf.py ../MiniCPM-V-4_6 --outfile ../MiniCPM-V-4_6/ggml-model-f16.gguf
-
-# multimodal projector (vision tower + window-attention vit_merger + DownsampleMLP merger)
-python ./convert_hf_to_gguf.py ../MiniCPM-V-4_6 --mmproj --outfile ../MiniCPM-V-4_6/mmproj-model-f16.gguf
-
-# optional: quantize to Q4_K_M
-./build/bin/llama-quantize ../MiniCPM-V-4_6/ggml-model-f16.gguf ../MiniCPM-V-4_6/ggml-model-Q4_K_M.gguf Q4_K_M
-```
-
-
-Inference on Linux or Mac
-```bash
-# run in single-turn mode
-./build/bin/llama-mtmd-cli -m ../MiniCPM-V-4_6/ggml-model-f16.gguf --mmproj ../MiniCPM-V-4_6/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?"
-
-# run in conversation mode
-./build/bin/llama-mtmd-cli -m ../MiniCPM-V-4_6/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-V-4_6/mmproj-model-f16.gguf
-```

docs/ops.md

@@ -17,7 +17,7 @@ Legend:
 |                              ABS | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                              ACC | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
 |                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             ADD1 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
+|                             ADD1 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           ARANGE | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
@@ -36,15 +36,15 @@ Legend:
 |                              CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
 |               CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |          CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-|                           CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             DIAG | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ |
+|                             DIAG | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ |
 |                    DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                              DIV | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                              DUP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                              ELU | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                              EXP | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                            EXPM1 | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ |
-|                             FILL | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             FILL | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ |
 |                   FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
 |                            FLOOR | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
 |                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | 🟡 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
@@ -101,11 +101,11 @@ Legend:
 |                         SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                         SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
-|                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ |
 |                              SQR | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
 |                             SQRT | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
 |                         SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
+|                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ |
 |                             STEP | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                              SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                              SUM | ❌ | 🟡 | ✅ | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |

examples/save-load-state/save-load-state.cpp

@@ -41,9 +41,6 @@ int main(int argc, char ** argv) {
     std::string result3;
 
     // init
-
-    ggml_backend_load_all();
-
     auto llama_init = common_init_from_params(params);
 
     auto * model = llama_init->model();

examples/speculative-simple/speculative-simple.cpp

@@ -4,6 +4,7 @@
 #include "speculative.h"
 #include "log.h"
 #include "llama.h"
+#include "chat.h"
 
 #include <clocale>
 #include <cstdio>
@@ -103,13 +104,53 @@ int main(int argc, char ** argv) {
             return 1;
         }
 
-        params.speculative.draft.model = model_dft.get();
+        params.speculative.draft.model_tgt = model_tgt;
+        params.speculative.draft.model     = model_dft.get();
         params.speculative.draft.cparams = common_context_params_to_llama(params_dft);
+
+        if (params.speculative.draft.eagle3) {
+            llama_set_eagle3(ctx_tgt, model_dft.get());
+        }
     }
 
+    // Apply chat template for EAGLE3 if available which can increase the acceptance rate
+    std::string prompt = params.prompt;
+    if (params.speculative.draft.eagle3) {
+        auto chat_templates = common_chat_templates_init(model_tgt, params.chat_template);
+        if (common_chat_templates_was_explicit(chat_templates.get())) {
+            std::vector<common_chat_msg> chat_msgs;
+            common_chat_msg user_msg;
+            user_msg.role = "user";
+            user_msg.content = params.prompt;
+            chat_msgs.push_back(user_msg);
+
+            common_chat_templates_inputs inputs;
+            inputs.messages = chat_msgs;
+            inputs.add_generation_prompt = true;
+            prompt = common_chat_templates_apply(chat_templates.get(), inputs).prompt;
+            LOG_INF("%s: EAGLE3 chat template applied\n", __func__);
+        }
+    }
+
+    int n_predict = 0;
+    int n_drafted = 0;
+    int n_accept  = 0;
+
+    // used to determine end of generation
+    bool has_eos = false;
+
+    // ================================================
+    // everything until here is standard initialization
+    // the relevant stuff for speculative decoding starts here
+
+    const auto t_enc_start = ggml_time_us();
+
+    // target model sampling context
+    common_sampler_ptr smpl(common_sampler_init(model_tgt, params.sampling));
+
     // Tokenize the prompt
     std::vector<llama_token> inp;
-    inp = common_tokenize(ctx_tgt, params.prompt, true, true);
+    inp = common_tokenize(ctx_tgt, prompt, true, true);
 
     if (llama_n_ctx(ctx_tgt) < (uint32_t) inp.size()) {
         LOG_ERR("%s: the prompt exceeds the context size (%d tokens, ctx %d)\n", __func__, (int) inp.size(), llama_n_ctx(ctx_tgt));
@@ -129,33 +170,39 @@ int main(int argc, char ** argv) {
         LOG("%s", common_token_to_piece(ctx_tgt, id).c_str());
     }
 
-    int n_predict = 0;
-    int n_drafted = 0;
-    int n_accept  = 0;
-
-    // used to determine end of generation
-    bool has_eos = false;
-
-    // ================================================
-    // everything until here is standard initialization
-    // the relevant stuff for speculative decoding starts here
-
-    const auto t_enc_start = ggml_time_us();
-
-    // target model sampling context
-    common_sampler_ptr smpl(common_sampler_init(model_tgt, params.sampling));
 
     // eval the prompt
-    llama_decode(ctx_tgt, llama_batch_get_one(inp.data(), inp.size() - 1));
+    llama_token id_last;
+    llama_tokens prompt_tgt;
+    int n_past;
 
-    // note: keep the last token separate!
-    llama_token id_last = inp.back();
+    // TODO: simplify
+    if (params.speculative.draft.eagle3) {
+        // Target model decodes full prompt and sample first token and intermediate features are extracted
+        llama_decode(ctx_tgt, llama_batch_get_one(inp.data(), inp.size()));
 
-    // all tokens currently in the target context
-    llama_tokens prompt_tgt(inp.begin(), inp.end() - 1);
-    prompt_tgt.reserve(llama_n_ctx(ctx_tgt));
+        id_last = common_sampler_sample(smpl.get(), ctx_tgt, -1);
+        common_sampler_accept(smpl.get(), id_last, true);
+        LOG("%s", common_token_to_piece(ctx_tgt, id_last).c_str());
+        n_predict++;
 
-    int n_past = inp.size() - 1;
+        // all tokens currently in the target context
+        prompt_tgt.assign(inp.begin(), inp.end());
+        prompt_tgt.reserve(llama_n_ctx(ctx_tgt));
+
+        n_past = inp.size();
+    } else {
+        llama_decode(ctx_tgt, llama_batch_get_one(inp.data(), inp.size() - 1));
+
+        // note: keep the last token separate!
+        id_last = inp.back();
+
+        // all tokens currently in the target context
+        prompt_tgt.assign(inp.begin(), inp.end() - 1);
+        prompt_tgt.reserve(llama_n_ctx(ctx_tgt));
+
+        n_past = inp.size() - 1;
+    }
 
     // init the speculator
     const auto & params_spec = params.speculative;

examples/sycl/start-svr.sh

@@ -111,14 +111,14 @@ if [ $GGML_SYCL_DEVICE -ne -1 ]; then
     echo "Use $GGML_SYCL_DEVICE as main GPU"
     #use signle GPU only
     GPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}"
-    export ONEAPI_DEVICE_SELECTOR="level_zero:${GGML_SYCL_DEVICE}"
+    export ONEAPI_DEVICE_SELECTOR="level_zero:${$GGML_SYCL_DEVICE}"
     echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"
 else
     echo "Use all Intel GPUs, including iGPU & dGPU"
     GPUS_SETTING="-sm ${SPLIT_MODE}"
  fi
 
-echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE}  --mmap --host 0.0.0.0 --port 8000"
+echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE}  --mmap "
 ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap --host 0.0.0.0 --port 8000
 
 

examples/sycl/test.sh

@@ -119,7 +119,7 @@ if [ $GGML_SYCL_DEVICE -ne -1 ]; then
     echo "Use $GGML_SYCL_DEVICE as main GPU"
     #use signle GPU only
     GPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}"
-    export ONEAPI_DEVICE_SELECTOR="level_zero:${GGML_SYCL_DEVICE}"
+    export ONEAPI_DEVICE_SELECTOR="level_zero:${$GGML_SYCL_DEVICE}"
     echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"
 else
     echo "Use all Intel GPUs, including iGPU & dGPU"

ggml/CMakeLists.txt

@@ -4,8 +4,8 @@ project("ggml" C CXX ASM)
 
 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 11)
-set(GGML_VERSION_PATCH 0)
+set(GGML_VERSION_MINOR 10)
+set(GGML_VERSION_PATCH 2)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")

ggml/include/ggml-backend.h

@@ -169,7 +169,7 @@ extern "C" {
         // device type
         enum ggml_backend_dev_type type;
         // device id
-        //   for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0")
+        //   for PCI devices, this should be the PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:01:00.0")
         //   if the id is unknown, this should be NULL
         const char * device_id;
         // device capabilities

ggml/src/ggml-backend.cpp

@@ -965,7 +965,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str
         }
         if (sched->debug > 1) {
             ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);
-            GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_desc(node), node->name,
+            GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_name(node->op), node->name,
                 fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node),
                 graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)], node->flags & GGML_TENSOR_FLAG_COMPUTE ? 1 : 0);
             for (int j = 0; j < GGML_MAX_SRC; j++) {

ggml/src/ggml-cpu/arch-fallback.h

@@ -203,6 +203,7 @@
 #elif defined(__riscv)
 // quants.c
 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
+#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0
 // repack.cpp
 #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1
 #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4

ggml/src/ggml-cpu/arch/riscv/quants.c

@@ -480,104 +480,6 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
 #endif
 }
 
-#if defined(__riscv_v)
-static NOINLINE void ggml_vec_dot_q1_0_q8_0_vl256(const int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy) {
-    const int qk = QK1_0;
-    const int nb = n / qk;
-    assert(n % qk == 0);
-
-    const block_q1_0 * GGML_RESTRICT x = vx;
-    const block_q8_0 * GGML_RESTRICT y = vy;
-
-    //LMUL = 1, VLMAX = 32
-    const size_t vl32 = __riscv_vsetvl_e8m1(32);
-    assert(vl32 == 32);
-
-    const vint16m1_t zero = __riscv_vmv_v_x_i16m1(0, 1);
-
-    float sumf = 0;
-
-    for (int ib = 0; ib < nb; ++ib) {
-        const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);
-
-        float acc = 0;
-
-        for (int k = 0; k < 4; ++k) {
-            const block_q8_0 * GGML_RESTRICT yb = &y[ib * 4 + k];
-            const vbool8_t is_not_zero = __riscv_vlm_v_b8(x[ib].qs + 4 * k, vl32);
-
-            const vint8m1_t qy = __riscv_vle8_v_i8m1(yb->qs, vl32);
-            const vint8m1_t neg_qy = __riscv_vneg_v_i8m1(qy, vl32);
-            const vint8m1_t sy = __riscv_vmerge_vvm_i8m1(neg_qy, qy, is_not_zero, vl32);
-
-            const vint16m1_t red = __riscv_vwredsum_vs_i8m1_i16m1(sy, zero, vl32);
-            acc += GGML_CPU_FP16_TO_FP32(yb->d) * (float)__riscv_vmv_x_s_i16m1_i16(red);
-        }
-
-        sumf += d0 * acc;
-    }
-
-    *s = sumf;
-}
-
-static NOINLINE void ggml_vec_dot_q1_0_q8_0_vl128(const int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy) {
-    const int qk = QK1_0;
-    const int nb = n / qk;
-    assert(n % qk == 0);
-
-    const block_q1_0 * GGML_RESTRICT x = vx;
-    const block_q8_0 * GGML_RESTRICT y = vy;
-
-    //LMUL = 2, VLMAX = 32
-    const size_t vl32 = __riscv_vsetvl_e8m2(32);
-    assert(vl32 == 32);
-
-    const vint16m1_t zero = __riscv_vmv_v_x_i16m1(0, 1);
-
-    float sumf = 0;
-
-    for (int ib = 0; ib < nb; ++ib) {
-        const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);
-
-        float acc = 0;
-
-        for (int k = 0; k < 4; ++k) {
-            const block_q8_0 * GGML_RESTRICT yb = &y[ib * 4 + k];
-            const vbool4_t is_not_zero = __riscv_vlm_v_b4(x[ib].qs + 4 * k, vl32);
-
-            const vint8m2_t qy = __riscv_vle8_v_i8m2(yb->qs, vl32);
-            const vint8m2_t neg_qy =__riscv_vneg_v_i8m2(qy, vl32);
-            const vint8m2_t sy = __riscv_vmerge_vvm_i8m2(neg_qy, qy, is_not_zero, vl32);
-
-            const vint16m1_t red = __riscv_vwredsum_vs_i8m2_i16m1(sy, zero, vl32);
-            acc += GGML_CPU_FP16_TO_FP32(yb->d) * (float)__riscv_vmv_x_s_i16m1_i16(red);
-        }
-
-        sumf += d0 * acc;
-    }
-
-    *s = sumf;
-}
-#endif
-
-void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
-#if defined(__riscv_v)
-    assert(nrc == 1);
-
-    const size_t vlen_bits = __riscv_vlenb() * 8;
-
-    if (vlen_bits >= 256) {
-        ggml_vec_dot_q1_0_q8_0_vl256(n, s, vx, vy);
-    } else if (vlen_bits >= 128) {
-        ggml_vec_dot_q1_0_q8_0_vl128(n, s, vx, vy);
-    } else {
-        ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
-    }
-#else
-    ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
-#endif
-}
-
 void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
     assert(nrc == 1);
     UNUSED(nrc);

ggml/src/ggml-cpu/ggml-cpu.c

@@ -2965,45 +2965,6 @@ struct ggml_cplan ggml_graph_plan(
     return cplan;
 }
 
-
-// Try to fuse the current node with subsequent nodes for better performance.
-// Returns the number of nodes skipped by fusion (>=1), or 0 if no fusion was applied.
-static bool ggml_cpu_disable_fusion = false;  // initialized once in ggml_cpu_init(), read-only afterwards
-
-static int ggml_cpu_try_fuse_ops(
-        const struct ggml_cgraph * cgraph,
-        const int node_n,
-        const struct ggml_compute_params * params,
-        const struct ggml_cplan * cplan) {
-
-    if (ggml_cpu_disable_fusion || cplan->use_ref) {
-        return 0;
-    }
-
-    struct ggml_tensor * node = cgraph->nodes[node_n];
-
-    if (node->op == GGML_OP_RMS_NORM) {
-        // RMS_NORM + MUL fusion
-        const enum ggml_op fuse_ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL };
-        if (ggml_can_fuse(cgraph, node_n, fuse_ops, 2)) {
-            struct ggml_tensor * mul_node = cgraph->nodes[node_n + 1];
-            const struct ggml_tensor * mul_w = (mul_node->src[0] == node)
-                ? mul_node->src[1] : mul_node->src[0];
-            if (node->src[0]->type  == GGML_TYPE_F32 &&
-                mul_node->type      == GGML_TYPE_F32 &&
-                mul_w->type         == GGML_TYPE_F32 &&
-                mul_w->ne[0]        == node->ne[0]   &&
-                mul_w->nb[0]        == sizeof(float)) {
-
-                ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node);
-                return 1;
-            }
-        }
-    }
-
-    return 0;
-}
-
 static thread_ret_t ggml_graph_compute_thread(void * data) {
     struct ggml_compute_state * state = (struct ggml_compute_state *) data;
     struct ggml_threadpool    * tp    = state->threadpool;
@@ -3040,14 +3001,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
             continue;
         }
 
-        // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time
-        // Try fused ops, fall back to normal compute
-        const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, &params, cplan);
-        if (n_fused > 0) {
-            node_n += n_fused;
-        } else {
-            ggml_compute_forward(&params, node);
-        }
+        ggml_compute_forward(&params, node);
 
         if (state->ith == 0 && cplan->abort_callback &&
                 cplan->abort_callback(cplan->abort_callback_data)) {
@@ -3809,11 +3763,6 @@ void ggml_cpu_init(void) {
         ggml_init_riscv_arch_features();
 #endif
 
-        {
-            const char * env = getenv("GGML_CPU_DISABLE_FUSION");
-            ggml_cpu_disable_fusion = (env != NULL && atoi(env) == 1);
-        }
-
         is_first_call = false;
     }
 

ggml/src/ggml-cpu/ops.cpp

@@ -3713,27 +3713,11 @@ void ggml_compute_forward_norm(
 
 // ggml_compute_forward_group_rms_norm
 
-// fusion kinds that can be combined with the rms_norm computation in a single pass.
-// extend this enum when adding new fused variants (e.g. FUSE_ADD, FUSE_MUL_ADD, ...).
-enum ggml_rms_norm_fuse_op {
-    GGML_RMS_NORM_FUSE_OP_NONE,
-    GGML_RMS_NORM_FUSE_OP_MUL,
-};
-
-template <ggml_rms_norm_fuse_op FUSE_OP>
 static void ggml_compute_forward_rms_norm_f32(
         const ggml_compute_params * params,
-        ggml_tensor * dst_rms_norm,
-        ggml_tensor * dst_fused = nullptr) {
+        ggml_tensor * dst) {
 
-    const ggml_tensor * src0 = dst_rms_norm->src[0];
-    const ggml_tensor * src1 = nullptr;
-    ggml_tensor       * dst  = dst_rms_norm;
-
-    if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_OP_MUL) {
-        src1 = (dst_fused->src[0] == dst_rms_norm) ? dst_fused->src[1] : dst_fused->src[0];
-        dst  = dst_fused;
-    }
+    const ggml_tensor * src0 = dst->src[0];
 
     GGML_ASSERT(ggml_are_same_shape(src0, dst));
 
@@ -3742,10 +3726,11 @@ static void ggml_compute_forward_rms_norm_f32(
     const int ith = params->ith;
     const int nth = params->nth;
 
-    GGML_TENSOR_BINARY_OP_LOCALS
+    GGML_TENSOR_UNARY_OP_LOCALS
 
     float eps;
-    memcpy(&eps, dst_rms_norm->op_params, sizeof(float));
+    memcpy(&eps, dst->op_params, sizeof(float));
+
     GGML_ASSERT(eps >= 0.0f);
 
     // TODO: optimize
@@ -3755,32 +3740,25 @@ static void ggml_compute_forward_rms_norm_f32(
                 const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
 
                 ggml_float sum = 0.0;
-                // worth switching to explicit SIMD?
                 for (int64_t i00 = 0; i00 < ne00; i00++) {
                     sum += (ggml_float)(x[i00] * x[i00]);
                 }
 
-                const float mean  = sum/ne00;
+                const float mean = sum/ne00;
+
+                float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
+
+                memcpy(y, x, ne00 * sizeof(float));
+                // for (int i00 = 0; i00 < ne00; i00++) {
+                //     y[i00] = x[i00];
+                // }
+
                 const float scale = 1.0f/sqrtf(mean + eps);
 
                 // if you hit this, likely you got an inf somewhere earlier
                 assert(scale > 0.0f);
 
-                float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
-
-                if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_OP_MUL) {
-                    const int64_t i11 = i01 % ne11;
-                    const int64_t i12 = i02 % ne12;
-                    const int64_t i13 = i03 % ne13;
-                    const float * w = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13);
-
-                    for (int64_t i00 = 0; i00 < ne00; i00++) {
-                        y[i00] = x[i00] * scale * w[i00];
-                    }
-                } else {
-                    memcpy(y, x, ne00 * sizeof(float));
-                    ggml_vec_scale_f32(ne00, y, scale);
-                }
+                ggml_vec_scale_f32(ne00, y, scale);
             }
         }
     }
@@ -3795,31 +3773,7 @@ void ggml_compute_forward_rms_norm(
     switch (src0->type) {
         case GGML_TYPE_F32:
             {
-                ggml_compute_forward_rms_norm_f32<GGML_RMS_NORM_FUSE_OP_NONE>(params, dst);
-            } break;
-        default:
-            {
-                GGML_ABORT("fatal error");
-            }
-    }
-}
-
-// Fused RMS_NORM + MUL: computes dst = rms_norm(src0) * src1 in a single pass.
-// This avoids materializing the intermediate rms_norm result in memory.
-void ggml_compute_forward_rms_norm_mul_fused(
-        const ggml_compute_params * params,
-        ggml_tensor * dst_rms_norm,
-        ggml_tensor * dst_mul) {
-
-    GGML_ASSERT(dst_mul != nullptr);
-    GGML_ASSERT(dst_mul->src[0] == dst_rms_norm || dst_mul->src[1] == dst_rms_norm);
-
-    const ggml_tensor * src0 = dst_rms_norm->src[0];
-
-    switch (src0->type) {
-        case GGML_TYPE_F32:
-            {
-                ggml_compute_forward_rms_norm_f32<GGML_RMS_NORM_FUSE_OP_MUL>(params, dst_rms_norm, dst_mul);
+                ggml_compute_forward_rms_norm_f32(params, dst);
             } break;
         default:
             {

ggml/src/ggml-cpu/ops.h

@@ -44,7 +44,6 @@ void ggml_compute_forward_concat(const struct ggml_compute_params * params, stru
 void ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
 void ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
 void ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
-void ggml_compute_forward_rms_norm_mul_fused(const struct ggml_compute_params * params, struct ggml_tensor * dst_rms_norm, struct ggml_tensor * dst_mul);
 void ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
 void ggml_compute_forward_group_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
 void ggml_compute_forward_l2_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);

ggml/src/ggml-cuda/fattn-mma-f16.cuh

@@ -61,11 +61,6 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2,  64,  64,  64,  64, 2, true);
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 128, 2,  64,  64,  64,  64, 2, true);
 
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128,  8,  64, 4,  64,  96,  64,  64, 2, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16,  64, 4,  32,  96,  64,  64, 2, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 128, 2,  32,  96,  64,  64, 2, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 128, 2,  32,  96,  64,  64, 2, true);
-
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8,  64, 4,  64, 128, 128, 128, 2, true);
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16,  64, 4,  32, 128, 128, 128, 2, true);
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  32, 128, 128, 128, 2, true);
@@ -1566,10 +1561,6 @@ static __global__ void flash_attn_ext_f16(
         NO_DEVICE_CODE;
         return;
     }
-    if (DKQ == 192 && ncols2 != 8 && ncols2 != 16) {
-        NO_DEVICE_CODE;
-        return;
-    }
 #ifdef VOLTA_MMA_AVAILABLE
     if (ncols1*ncols2 < 32) {
         NO_DEVICE_CODE;

ggml/src/ggml-cuda/fattn-tile.cu

@@ -34,10 +34,6 @@ void ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor
             GGML_ASSERT(V->ne[0] == K->ne[0]);
             ggml_cuda_flash_attn_ext_tile_case<128, 128>(ctx, dst);
         } break;
-        case 192: {
-            GGML_ASSERT(V->ne[0] == 128);
-            ggml_cuda_flash_attn_ext_tile_case<192, 128>(ctx, dst);
-        } break;
         case 256: {
             GGML_ASSERT(V->ne[0] == K->ne[0]);
             ggml_cuda_flash_attn_ext_tile_case<256, 256>(ctx, dst);

ggml/src/ggml-cuda/fattn-tile.cuh

@@ -62,12 +62,6 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 2,  64,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2,  64,  64)
 
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  2,  64, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  4, 128, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  8, 256, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2,  64,  64)
-
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2,  64, 2,  64,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 128, 2,  64,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 256, 2,  64,  64)
@@ -130,12 +124,6 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 128, 3,  32, 128)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2,  64,  64)
 
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  2, 128, 3,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  4, 128, 3,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  8, 256, 2,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2,  32,  64)
-
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2, 128, 3,  64,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 128, 3,  32,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 256, 2,  32, 256)
@@ -205,12 +193,6 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2,  64,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 2,  64,  32)
 
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  2, 256, 2, 128,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  4, 256, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  8, 256, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2,  32,  64)
-
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2, 256, 2, 128,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 256, 2,  64, 128)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 256, 2,  64, 128)
@@ -282,12 +264,6 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 3, 128,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 3,  64,  64)
 
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  2,  64, 8,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  4, 128, 6,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  8, 128, 6,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 5,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 3,  64,  64)
-
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2,  64, 8,  32,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 128, 6,  32, 256)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 128, 6,  32, 256)
@@ -1274,20 +1250,7 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm
         }
     }
 
-    if constexpr (DKQ == 192) {
-        // MiMo-V2.5 / V2.5-Pro / V2-Flash: gqa_ratio is 8 (SWA) or 16 (full attn)
-        if (use_gqa_opt && gqa_ratio % 16 == 0) {
-            launch_fattn_tile_switch_ncols1<DKQ, DV, 16, use_logit_softcap>(ctx, dst);
-            return;
-        }
-        if (use_gqa_opt && gqa_ratio % 8 == 0) {
-            launch_fattn_tile_switch_ncols1<DKQ, DV,  8, use_logit_softcap>(ctx, dst);
-            return;
-        }
-        GGML_ABORT("flash-attn tile (192/128): expected GQA ratio multiple of 8");
-    }
-
-    if constexpr (DKQ <= 512 && DKQ != 320 && DKQ != 192) {
+    if constexpr (DKQ <= 512 && DKQ != 320) {
         if (use_gqa_opt && gqa_ratio % 8 == 0) {
             launch_fattn_tile_switch_ncols1<DKQ, DV, 8, use_logit_softcap>(ctx, dst);
             return;
@@ -1340,7 +1303,6 @@ extern DECL_FATTN_TILE_CASE( 80,  80);
 extern DECL_FATTN_TILE_CASE( 96,  96);
 extern DECL_FATTN_TILE_CASE(112, 112);
 extern DECL_FATTN_TILE_CASE(128, 128);
-extern DECL_FATTN_TILE_CASE(192, 128);
 extern DECL_FATTN_TILE_CASE(256, 256);
 extern DECL_FATTN_TILE_CASE(320, 256);
 extern DECL_FATTN_TILE_CASE(512, 512);

ggml/src/ggml-cuda/fattn.cu

@@ -139,22 +139,6 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
             GGML_ASSERT(V->ne[0] == 128);
             ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<128, 128>(ctx, dst);
             break;
-        case 192: {
-            // MiMo-V2.5 / V2.5-Pro / V2-Flash: gqa_ratio is 8 (SWA) or 16 (full attn)
-            GGML_ASSERT(V->ne[0] == 128);
-            float max_bias = 0.0f;
-            memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));
-            const bool use_gqa_opt = mask && max_bias == 0.0f;
-            GGML_ASSERT(use_gqa_opt);
-            GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);
-            const int gqa_ratio = Q->ne[2] / K->ne[2];
-            if (gqa_ratio % 16 == 0) {
-                ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<192, 128, 16>(ctx, dst);
-            } else {
-                GGML_ASSERT(gqa_ratio % 8 == 0);
-                ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<192, 128,  8>(ctx, dst);
-            }
-        } break;
         case 256:
             GGML_ASSERT(V->ne[0] == 256);
             ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<256, 256>(ctx, dst);
@@ -384,14 +368,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
                 return BEST_FATTN_KERNEL_NONE;
             }
             break;
-        case 192:
-            if (V->ne[0] != 128 || !gqa_opt_applies) {
-                return BEST_FATTN_KERNEL_NONE;
-            }
-            if (gqa_ratio % 8 != 0) {
-                return BEST_FATTN_KERNEL_NONE;
-            }
-            break;
         case 320:
             if (V->ne[0] != 256 || !gqa_opt_applies) {
                 return BEST_FATTN_KERNEL_NONE;
@@ -449,8 +425,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
     }
 
     // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes:
-    // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path.
-    const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0;
+    const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0;
 
     // If Turing tensor cores are available, use them:
     if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
@@ -479,7 +454,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
 
     if (volta_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
         int gqa_ratio_eff = 1;
-        const int ncols2_max = (Q->ne[0] == 576 || Q->ne[0] == 192) ? 16 : 8;
+        const int ncols2_max = Q->ne[0] == 576 ? 16 : 8;
         while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
             gqa_ratio_eff *= 2;
         }
@@ -493,7 +468,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
     }
 
     // Use the WMMA kernel if possible:
-    if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 192 && Q->ne[0] != 512 && Q->ne[0] != 576) {
+    if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 512 && Q->ne[0] != 576) {
         if (can_use_vector_kernel && Q->ne[1] <= 2) {
             return BEST_FATTN_KERNEL_VEC;
         }
@@ -526,7 +501,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
     }
 
     // Use MFMA flash attention for CDNA (MI100+):
-    if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 192 && Q->ne[0] != 256 && Q->ne[0] != 512 && Q->ne[0] != 576) {
+    if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 256 && Q->ne[0] != 512 && Q->ne[0] != 576) {
         const int64_t eff_nq = Q->ne[1] * (gqa_opt_applies ? gqa_ratio : 1);
         // MMA vs tile crossover benchmarked on MI300X @ d32768:
         //   hsk=64  (gqa=4): MMA wins at eff >= 128 (+11%)

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -39,7 +39,6 @@
 #include "ggml-cuda/rope.cuh"
 #include "ggml-cuda/roll.cuh"
 #include "ggml-cuda/scale.cuh"
-#include "ggml-cuda/snake.cuh"
 #include "ggml-cuda/softcap.cuh"
 #include "ggml-cuda/softmax.cuh"
 #include "ggml-cuda/ssm-conv.cuh"
@@ -3758,35 +3757,6 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
         return 2;
     }
 
-    // Snake activation: y = x + sin(a*x)^2 * inv_b
-    // Naive 5-op decomposition emitted by frontends: mul -> sin -> sqr -> mul -> add
-    if (ggml_can_fuse_subgraph(cgraph, i,
-            { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD },
-            { i + 4 })) {
-        const ggml_tensor * mul0 = cgraph->nodes[i];
-        const ggml_tensor * sqr  = cgraph->nodes[i + 2];
-        const ggml_tensor * mul1 = cgraph->nodes[i + 3];
-        ggml_tensor *       add  = cgraph->nodes[i + 4];
-
-        // x carries the full activation shape, a is the broadcast operand
-        const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1];
-        const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0];
-
-        // mul1 reads sqr and inv_b in either operand order
-        const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0];
-
-        // closure check: the trailing add must read the same x as the leading mul
-        const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0];
-
-        const bool type_ok  = (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16);
-        const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1];
-
-        if (type_ok && shape_ok && x_in_add == x && add->type == x->type) {
-            ggml_cuda_op_snake_fused(*cuda_ctx, x, a, inv_b, add);
-            return 4;
-        }
-    }
-
     // multi-(add or mul)
     if (node->op == GGML_OP_ADD || node->op == GGML_OP_MUL) {
         int     n_fuse = 0;
@@ -5464,9 +5434,6 @@ ggml_backend_reg_t ggml_backend_cuda_reg() {
                 char pci_bus_id[32] = {};
                 CUDA_CHECK(cudaDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), i));
                 dev_ctx->pci_bus_id = pci_bus_id;
-                for (char & c : dev_ctx->pci_bus_id) {
-                    c = std::tolower(c);
-                }
                 dev_ctx->op_offload_min_batch_size = min_batch_size;
 
                 ggml_backend_dev_t dev = new ggml_backend_device {

ggml/src/ggml-cuda/out-prod.cu

@@ -54,31 +54,15 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     const int64_t dps2 = ne2 / ne02;
     const int64_t dps3 = ne3 / ne03;
 
-    if (dps2 == 1 && ne2 > 1) {
-        // src0 has uniform stride s02 along dim 2; batch the inner loop with a strided GEMM
-        GGML_ASSERT(ne2 <= std::numeric_limits<int>::max());
-        const int batch_count = (int) ne2;
-        for (int64_t i3 = 0; i3 < ne3; ++i3) {
+    // TODO batched matrix multiplication
+    for (int64_t i3 = 0; i3 < ne3; ++i3) {
+        for (int64_t i2 = 0; i2 < ne2; ++i2) {
             CUBLAS_CHECK(
-                cublasSgemmStridedBatched(handle, CUBLAS_OP_N, src1_cublas_op,
+                cublasSgemm(handle, CUBLAS_OP_N, src1_cublas_op,
                         ne0, ne1, ne01,
-                        &alpha, src0_d + (i3/dps3)*s03, lda, s02,
-                                src1_d +  i3     *s13, ldb, s12,
-                        &beta,  dst_d  +  i3     *s3,  ldc, s2,
-                        batch_count));
-        }
-    } else {
-        // Fallback: ne2 == 1 (no batching benefit) or dps2 > 1 (src0 broadcast along dim 2
-        // with non-uniform stride; would need cublasSgemmBatched with pointer arrays).
-        for (int64_t i3 = 0; i3 < ne3; ++i3) {
-            for (int64_t i2 = 0; i2 < ne2; ++i2) {
-                CUBLAS_CHECK(
-                    cublasSgemm(handle, CUBLAS_OP_N, src1_cublas_op,
-                            ne0, ne1, ne01,
-                            &alpha, src0_d + (i3/dps3)*s03 + (i2/dps2)*s02, lda,
-                                    src1_d +  i3      *s13 +  i2      *s12, ldb,
-                            &beta,  dst_d  +  i3      *s3  +  i2      *s2,  ldc));
-            }
+                        &alpha, src0_d + (i3/dps3)*s03 + (i2/dps2)*s02, lda,
+                                src1_d +  i3      *s13 +  i2      *s12, ldb,
+                        &beta,  dst_d  +  i3      *s3  +  i2      *s2,  ldc));
         }
     }
 }

ggml/src/ggml-cuda/snake.cu

@@ -1,72 +0,0 @@
-#include "snake.cuh"
-#include "convert.cuh"
-
-// Fused Snake activation: y = x + sin^2(a * x) * inv_b
-// x: [T, C] (T contiguous), a: [1, C], inv_b: [1, C]
-// Supports F32, F16, BF16 data with F32 compute.
-
-template <typename T>
-static __global__ void snake_kernel(
-        const T     * __restrict__ x,
-        const float * __restrict__ a,
-        const float * __restrict__ inv_b,
-        T           * __restrict__ dst,
-        const int    total,
-        const uint3  T_len_fastdiv) {
-    const int idx = blockIdx.x * blockDim.x + threadIdx.x;
-    if (idx >= total) return;
-
-    const int c = (int) fastdiv((uint32_t) idx, T_len_fastdiv);
-
-    const float xi = ggml_cuda_cast<float>(x[idx]);
-    const float s  = sinf(a[c] * xi);
-    dst[idx] = ggml_cuda_cast<T>(xi + s * s * inv_b[c]);
-}
-
-// Internal launcher with explicit x/a/inv_b/dst tensors.
-// Shared by the public op (reads dst->src) and the fusion path (explicit args).
-static void launch_snake(ggml_backend_cuda_context & ctx,
-                         const ggml_tensor * x,
-                         const ggml_tensor * a,
-                         const ggml_tensor * inv_b,
-                         ggml_tensor *       dst) {
-    const float * a_d     = (const float *)a->data;
-    const float * inv_b_d = (const float *)inv_b->data;
-
-    const int   T = (int)x->ne[0];
-    const int   C = (int)x->ne[1];
-    const int   total = T * C;
-    const uint3 T_len_fastdiv = init_fastdiv_values((uint64_t) T);
-
-    const int block_size = 256;
-    const int grid_size  = (total + block_size - 1) / block_size;
-
-    cudaStream_t stream = ctx.stream();
-
-    switch (x->type) {
-        case GGML_TYPE_F32: {
-            snake_kernel<<<grid_size, block_size, 0, stream>>>(
-                (const float *)x->data, a_d, inv_b_d, (float *)dst->data, total, T_len_fastdiv);
-        } break;
-        case GGML_TYPE_F16: {
-            snake_kernel<<<grid_size, block_size, 0, stream>>>(
-                (const half *)x->data, a_d, inv_b_d, (half *)dst->data, total, T_len_fastdiv);
-        } break;
-        case GGML_TYPE_BF16: {
-            snake_kernel<<<grid_size, block_size, 0, stream>>>(
-                (const nv_bfloat16 *)x->data, a_d, inv_b_d, (nv_bfloat16 *)dst->data, total, T_len_fastdiv);
-        } break;
-        default:
-            GGML_ABORT("snake: unsupported type");
-    }
-}
-
-// Fusion entry: caller supplies x/a/inv_b explicitly from the matched
-// mul -> sin -> sqr -> mul -> add pattern. The dst is the trailing add output.
-void ggml_cuda_op_snake_fused(ggml_backend_cuda_context & ctx,
-                              const ggml_tensor * x,
-                              const ggml_tensor * a,
-                              const ggml_tensor * inv_b,
-                              ggml_tensor *       dst) {
-    launch_snake(ctx, x, a, inv_b, dst);
-}

ggml/src/ggml-cuda/snake.cuh

@@ -1,8 +0,0 @@
-#include "common.cuh"
-
-// Fusion entry point. Caller supplies x/a/inv_b explicitly.
-void ggml_cuda_op_snake_fused(ggml_backend_cuda_context & ctx,
-                              const ggml_tensor * x,
-                              const ggml_tensor * a,
-                              const ggml_tensor * inv_b,
-                              ggml_tensor *       dst);

ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu

@@ -2,5 +2,4 @@
 
 #include "../fattn-mma-f16.cuh"
 
-DECL_FATTN_MMA_F16_CASE(192, 128, 1, 16);
 DECL_FATTN_MMA_F16_CASE(576, 512, 1, 16);

ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu

@@ -7,6 +7,5 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 1, 8);
 DECL_FATTN_MMA_F16_CASE(96, 96, 1, 8);
 DECL_FATTN_MMA_F16_CASE(112, 112, 1, 8);
 DECL_FATTN_MMA_F16_CASE(128, 128, 1, 8);
-DECL_FATTN_MMA_F16_CASE(192, 128, 1, 8);
 DECL_FATTN_MMA_F16_CASE(256, 256, 1, 8);
 DECL_FATTN_MMA_F16_CASE(512, 512, 1, 8);

ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu

@@ -2,5 +2,4 @@
 
 #include "../fattn-mma-f16.cuh"
 
-DECL_FATTN_MMA_F16_CASE(192, 128, 2, 16);
 DECL_FATTN_MMA_F16_CASE(576, 512, 2, 16);

ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu

@@ -7,6 +7,5 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 2, 8);
 DECL_FATTN_MMA_F16_CASE(96, 96, 2, 8);
 DECL_FATTN_MMA_F16_CASE(112, 112, 2, 8);
 DECL_FATTN_MMA_F16_CASE(128, 128, 2, 8);
-DECL_FATTN_MMA_F16_CASE(192, 128, 2, 8);
 DECL_FATTN_MMA_F16_CASE(256, 256, 2, 8);
 DECL_FATTN_MMA_F16_CASE(512, 512, 2, 8);

ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu

@@ -2,5 +2,4 @@
 
 #include "../fattn-mma-f16.cuh"
 
-DECL_FATTN_MMA_F16_CASE(192, 128, 4, 16);
 DECL_FATTN_MMA_F16_CASE(576, 512, 4, 16);

ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu

@@ -7,6 +7,5 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 4, 8);
 DECL_FATTN_MMA_F16_CASE(96, 96, 4, 8);
 DECL_FATTN_MMA_F16_CASE(112, 112, 4, 8);
 DECL_FATTN_MMA_F16_CASE(128, 128, 4, 8);
-DECL_FATTN_MMA_F16_CASE(192, 128, 4, 8);
 DECL_FATTN_MMA_F16_CASE(256, 256, 4, 8);
 DECL_FATTN_MMA_F16_CASE(512, 512, 4, 8);

ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu

@@ -7,6 +7,5 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 8, 8);
 DECL_FATTN_MMA_F16_CASE(96, 96, 8, 8);
 DECL_FATTN_MMA_F16_CASE(112, 112, 8, 8);
 DECL_FATTN_MMA_F16_CASE(128, 128, 8, 8);
-DECL_FATTN_MMA_F16_CASE(192, 128, 8, 8);
 DECL_FATTN_MMA_F16_CASE(256, 256, 8, 8);
 DECL_FATTN_MMA_F16_CASE(512, 512, 8, 8);

ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu

@@ -1,5 +0,0 @@
-// This file has been autogenerated by generate_cu_files.py, do not edit manually.
-
-#include "../fattn-tile.cuh"
-
-DECL_FATTN_TILE_CASE(192, 128);

ggml/src/ggml-cuda/template-instances/generate_cu_files.py

@@ -3,10 +3,7 @@
 from glob import glob
 import os
 
-HEAD_SIZES_KQ = [40, 64, 72, 80, 96, 112, 128, 192, 256, 320, 512, 576]
-
-# DKQ -> DV override for asymmetric head dims.
-HEAD_SIZES_V_OVERRIDE = {576: 512, 320: 256, 192: 128}
+HEAD_SIZES_KQ = [40, 64, 72, 80, 96, 112, 128, 256, 320, 512, 576]
 
 TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_BF16"]
 
@@ -65,7 +62,7 @@ for filename in glob("*.cu"):
     os.remove(filename)
 
 for head_size_kq in HEAD_SIZES_KQ:
-    head_size_v = HEAD_SIZES_V_OVERRIDE.get(head_size_kq, head_size_kq)
+    head_size_v = 256 if head_size_kq == 320 else (head_size_kq if head_size_kq != 576 else 512)
     with open(f"fattn-tile-instance-dkq{head_size_kq}-dv{head_size_v}.cu", "w") as f:
         f.write(SOURCE_FATTN_TILE.format(head_size_kq=head_size_kq, head_size_v=head_size_v))
 
@@ -88,17 +85,15 @@ for ncols in [8, 16, 32, 64]:
                 if head_size_kq == 72:
                     continue
                 # Skip compilation of unused ncols2 values for niche head sizes:
-                if head_size_kq == 192 and ncols2 not in (8, 16): # MiMo-V2.5
-                    continue
                 if head_size_kq == 320 and ncols2 != 32: # Mistral Small 4
                     continue
                 if head_size_kq == 512 and ncols2 not in (4, 8): # Gemma 4
                     continue
                 if head_size_kq == 576 and ncols2 not in (4, 16, 32): # Deepseek, GLM 4.7 Flash
                     continue
-                if head_size_kq not in (192, 320, 576) and ncols2 in (16, 32):
+                if head_size_kq not in (320, 576) and ncols2 in (16, 32):
                     continue
-                head_size_v = HEAD_SIZES_V_OVERRIDE.get(head_size_kq, head_size_kq)
+                head_size_v = 256 if head_size_kq == 320 else (head_size_kq if head_size_kq != 576 else 512)
                 f.write(SOURCE_FATTN_MMA_CASE.format(ncols1=ncols1, ncols2=ncols2, head_size_kq=head_size_kq, head_size_v=head_size_v))
 
 for type in TYPES_MMQ:

ggml/src/ggml-cuda/vendors/hip.h

@@ -48,7 +48,6 @@
 #define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS
 #define cublasSetStream hipblasSetStream
 #define cublasSgemm hipblasSgemm
-#define cublasSgemmStridedBatched hipblasSgemmStridedBatched
 #define cublasStatus_t hipblasStatus_t
 #define cublasOperation_t hipblasOperation_t
 #define cudaDevAttrCooperativeLaunch hipDeviceAttributeCooperativeLaunch

ggml/src/ggml-cuda/vendors/musa.h

@@ -32,7 +32,6 @@
 #define cublasSetMathMode mublasSetMathMode
 #define cublasSetStream mublasSetStream
 #define cublasSgemm mublasSgemm
-#define cublasSgemmStridedBatched mublasSgemmStridedBatched
 #define cublasStatus_t mublasStatus_t
 #define cublasOperation_t mublasOperation_t
 #define cublasGetStatusString mublasGetStatusString

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -2261,58 +2261,6 @@ static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_sess
     return true;
 }
 
-static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
-    const struct ggml_tensor * q     = op->src[0];
-    const struct ggml_tensor * k     = op->src[1];
-    const struct ggml_tensor * v     = op->src[2];
-    const struct ggml_tensor * g     = op->src[3];
-    const struct ggml_tensor * beta  = op->src[4];
-    const struct ggml_tensor * state = op->src[5];
-    const struct ggml_tensor * dst   = op;
-
-    if (!q || !k || !v || !g || !beta || !state) {
-        return false;
-    }
-
-    if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 ||
-        g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 ||
-        dst->type != GGML_TYPE_F32) {
-        return false;
-    }
-
-    if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) ||
-        !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) ||
-        !ggml_is_contiguous(dst)) {
-        return false;
-    }
-
-    const int64_t S_v      = v->ne[0];
-    const int64_t H        = v->ne[1];
-    const int64_t n_tokens = v->ne[2];
-    const int64_t n_seqs   = v->ne[3];
-
-    if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) {
-        return false;
-    }
-    if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 ||
-        q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 ||
-        (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {
-        return false;
-    }
-    if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {
-        return false;
-    }
-    if (ggml_nelements(state) != S_v * S_v * H * n_seqs) {
-        return false;
-    }
-    if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs) {
-        return false;
-    }
-
-    GGML_UNUSED(sess);
-    return true;
-}
-
 static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {
     const struct ggml_tensor * src0 = dst->src[0];
     const struct ggml_tensor * src1 = dst->src[1];
@@ -2472,8 +2420,8 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses
         return false;
     }
 
-    // dst must be contiguous; src0 may be non-contiguous
-    if (!ggml_is_contiguous(dst)) {
+    // TODO: add support for non-contiguous elements within a row
+    if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(dst)) {
         return false;
     }
 
@@ -2829,34 +2777,32 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) {
 
 static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
     switch (t->op) {
-        case GGML_OP_FLASH_ATTN_EXT:  return HTP_OP_FLASH_ATTN_EXT;
-        case GGML_OP_MUL_MAT:         return HTP_OP_MUL_MAT;
-        case GGML_OP_MUL_MAT_ID:      return HTP_OP_MUL_MAT_ID;
-        case GGML_OP_MUL:             return HTP_OP_MUL;
-        case GGML_OP_ADD:             return HTP_OP_ADD;
-        case GGML_OP_ADD_ID:          return HTP_OP_ADD_ID;
-        case GGML_OP_SUB:             return HTP_OP_SUB;
-        case GGML_OP_DIV:             return HTP_OP_DIV;
-        case GGML_OP_CPY:             return HTP_OP_CPY;
-        case GGML_OP_CONT:            return HTP_OP_CPY;
-        case GGML_OP_GET_ROWS:        return HTP_OP_GET_ROWS;
-        case GGML_OP_SET_ROWS:        return HTP_OP_SET_ROWS;
-        case GGML_OP_SUM_ROWS:        return HTP_OP_SUM_ROWS;
-        case GGML_OP_ARGSORT:         return HTP_OP_ARGSORT;
-        case GGML_OP_L2_NORM:         return HTP_OP_L2_NORM;
-        case GGML_OP_RMS_NORM:        return HTP_OP_RMS_NORM;
-        case GGML_OP_SCALE:           return HTP_OP_SCALE;
-        case GGML_OP_SQR:             return HTP_OP_SQR;
-        case GGML_OP_SQRT:            return HTP_OP_SQRT;
-        case GGML_OP_SOFT_MAX:        return HTP_OP_SOFTMAX;
-        case GGML_OP_SSM_CONV:        return HTP_OP_SSM_CONV;
-        case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;
-        case GGML_OP_ROPE:            return HTP_OP_ROPE;
-        case GGML_OP_REPEAT:          return HTP_OP_REPEAT;
-        case GGML_OP_CUMSUM:          return HTP_OP_CUMSUM;
-        case GGML_OP_FILL:            return HTP_OP_FILL;
-        case GGML_OP_DIAG:            return HTP_OP_DIAG;
-        case GGML_OP_SOLVE_TRI:       return HTP_OP_SOLVE_TRI;
+        case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;
+        case GGML_OP_MUL_MAT:        return HTP_OP_MUL_MAT;
+        case GGML_OP_MUL_MAT_ID:     return HTP_OP_MUL_MAT_ID;
+        case GGML_OP_MUL:            return HTP_OP_MUL;
+        case GGML_OP_ADD:            return HTP_OP_ADD;
+        case GGML_OP_ADD_ID:         return HTP_OP_ADD_ID;
+        case GGML_OP_SUB:            return HTP_OP_SUB;
+        case GGML_OP_DIV:            return HTP_OP_DIV;
+        case GGML_OP_CPY:            return HTP_OP_CPY;
+        case GGML_OP_CONT:           return HTP_OP_CPY;
+        case GGML_OP_GET_ROWS:       return HTP_OP_GET_ROWS;
+        case GGML_OP_SET_ROWS:       return HTP_OP_SET_ROWS;
+        case GGML_OP_SUM_ROWS:       return HTP_OP_SUM_ROWS;
+        case GGML_OP_ARGSORT:        return HTP_OP_ARGSORT;
+        case GGML_OP_RMS_NORM:       return HTP_OP_RMS_NORM;
+        case GGML_OP_SCALE:          return HTP_OP_SCALE;
+        case GGML_OP_SQR:            return HTP_OP_SQR;
+        case GGML_OP_SQRT:           return HTP_OP_SQRT;
+        case GGML_OP_SOFT_MAX:       return HTP_OP_SOFTMAX;
+        case GGML_OP_SSM_CONV:       return HTP_OP_SSM_CONV;
+        case GGML_OP_ROPE:           return HTP_OP_ROPE;
+        case GGML_OP_REPEAT:         return HTP_OP_REPEAT;
+        case GGML_OP_CUMSUM:         return HTP_OP_CUMSUM;
+        case GGML_OP_FILL:           return HTP_OP_FILL;
+        case GGML_OP_DIAG:           return HTP_OP_DIAG;
+        case GGML_OP_SOLVE_TRI:      return HTP_OP_SOLVE_TRI;
         case GGML_OP_UNARY:
             switch (ggml_get_unary_op(t)) {
                 case GGML_UNARY_OP_SILU:     return HTP_OP_UNARY_SILU;
@@ -3307,10 +3253,6 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
             supp = ggml_hexagon_supported_add_id(sess, op);
             break;
 
-        case GGML_OP_L2_NORM:
-            supp = ggml_hexagon_supported_unary(sess, op);
-            break;
-
         case GGML_OP_RMS_NORM:
         case GGML_OP_SCALE:
             supp = ggml_hexagon_supported_unary(sess, op);
@@ -3394,10 +3336,6 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
             supp = ggml_hexagon_supported_ssm_conv(sess, op);
             break;
 
-        case GGML_OP_GATED_DELTA_NET:
-            supp = ggml_hexagon_supported_gated_delta_net(sess, op);
-            break;
-
         case GGML_OP_CUMSUM:
             supp = ggml_hexagon_supported_cumsum(sess, op);
             break;

ggml/src/ggml-hexagon/htp/CMakeLists.txt

@@ -37,7 +37,6 @@ add_library(${HTP_LIB} SHARED
     fill-ops.c
     diag-ops.c
     solve-tri-ops.c
-    gated-delta-net-ops.c
 )
 
 target_compile_definitions(${HTP_LIB} PRIVATE

ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c

@@ -1,955 +0,0 @@
-#include <math.h>
-#include <stdint.h>
-#include <string.h>
-
-#include "hvx-utils.h"
-
-#define GGML_COMMON_DECL_C
-#include "ggml-common.h"
-#include "htp-ctx.h"
-
-#ifndef MIN
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-#endif
-
-#define HTP_GDN_MAX_SV 128
-
-struct htp_gdn_context {
-    struct htp_ops_context * octx;
-    uint32_t rows_per_thread;
-    size_t state_bytes;
-    bool use_vtcm;
-    uint8_t * vtcm_state_base;
-    size_t vtcm_state_per_thread;
-};
-
-static inline float gdn_mul_dot_f32(float * restrict dst, const float * restrict mul,
-        const float * restrict dot, uint32_t n) {
-    HVX_Vector acc = Q6_V_vzero();
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vd = hvx_vmemu(dst + i * epv);
-        HVX_Vector vm = hvx_vmem(mul + i * epv);
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-        HVX_Vector out = hvx_vec_mul_f32_f32(vd, vm);
-        hvx_vmemu(dst + i * epv) = out;
-        acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vd = hvx_vmemu(dst + off);
-        HVX_Vector vm = hvx_vmem(mul + off);
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_Vector out = hvx_vec_mul_f32_f32(vd, vm);
-        hvx_vec_store_u(dst + off, tail * sizeof(float), out);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector prod = hvx_vec_mul_f32_f32(out, vdot);
-        acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, prod, Q6_V_vzero()));
-    }
-
-    return hvx_vec_get_f32(hvx_vec_reduce_sum_f32(acc));
-}
-
-static inline float gdn_mul_scalar_dot_f32(float * restrict dst, float mul,
-        const float * restrict dot, uint32_t n) {
-    HVX_Vector acc = Q6_V_vzero();
-    const HVX_Vector vmul = hvx_vec_splat_f32(mul);
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vd = hvx_vmemu(dst + i * epv);
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-        HVX_Vector out = hvx_vec_mul_f32_f32(vd, vmul);
-        hvx_vmemu(dst + i * epv) = out;
-        acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vd = hvx_vmemu(dst + off);
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_Vector out = hvx_vec_mul_f32_f32(vd, vmul);
-        hvx_vec_store_u(dst + off, tail * sizeof(float), out);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector prod = hvx_vec_mul_f32_f32(out, vdot);
-        acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, prod, Q6_V_vzero()));
-    }
-
-    return hvx_vec_get_f32(hvx_vec_reduce_sum_f32(acc));
-}
-
-static inline float gdn_add_scaled_dot_f32(float * restrict dst, const float * restrict src,
-        float scale, const float * restrict dot, uint32_t n) {
-    HVX_Vector acc = Q6_V_vzero();
-    const HVX_Vector vscale = hvx_vec_splat_f32(scale);
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vd = hvx_vmemu(dst + i * epv);
-        HVX_Vector vs = hvx_vmem(src + i * epv);
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-        HVX_Vector out = hvx_vec_add_f32_f32(vd, hvx_vec_mul_f32_f32(vs, vscale));
-        hvx_vmemu(dst + i * epv) = out;
-        acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vd = hvx_vmemu(dst + off);
-        HVX_Vector vs = hvx_vmem(src + off);
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_Vector out = hvx_vec_add_f32_f32(vd, hvx_vec_mul_f32_f32(vs, vscale));
-        hvx_vec_store_u(dst + off, tail * sizeof(float), out);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector prod = hvx_vec_mul_f32_f32(out, vdot);
-        acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, prod, Q6_V_vzero()));
-    }
-
-    return hvx_vec_get_f32(hvx_vec_reduce_sum_f32(acc));
-}
-
-static inline void gdn_mul_dot4_f32(float * restrict dst0, float * restrict dst1,
-        float * restrict dst2, float * restrict dst3, const float * restrict mul,
-        const float * restrict dot, uint32_t n, float * restrict sums) {
-    HVX_Vector acc0 = Q6_V_vzero();
-    HVX_Vector acc1 = Q6_V_vzero();
-    HVX_Vector acc2 = Q6_V_vzero();
-    HVX_Vector acc3 = Q6_V_vzero();
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vm = hvx_vmem(mul + i * epv);
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-
-        HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vm);
-        HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vm);
-        HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + i * epv), vm);
-        HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + i * epv), vm);
-
-        hvx_vmemu(dst0 + i * epv) = out0;
-        hvx_vmemu(dst1 + i * epv) = out1;
-        hvx_vmemu(dst2 + i * epv) = out2;
-        hvx_vmemu(dst3 + i * epv) = out3;
-
-        acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot));
-        acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot));
-        acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot));
-        acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vm = hvx_vmem(mul + off);
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector zero = Q6_V_vzero();
-
-        HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + off), vm);
-        HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + off), vm);
-        HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + off), vm);
-        HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + off), vm);
-
-        hvx_vec_store_u(dst0 + off, tail * sizeof(float), out0);
-        hvx_vec_store_u(dst1 + off, tail * sizeof(float), out1);
-        hvx_vec_store_u(dst2 + off, tail * sizeof(float), out2);
-        hvx_vec_store_u(dst3 + off, tail * sizeof(float), out3);
-
-        acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero));
-        acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero));
-        acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero));
-        acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero));
-    }
-
-    HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } };
-    hvx_vec_store_u(sums, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(acc));
-}
-
-static inline void gdn_mul_scalar_dot4_f32(float * restrict dst0, float * restrict dst1,
-        float * restrict dst2, float * restrict dst3, float mul,
-        const float * restrict dot, uint32_t n, float * restrict sums) {
-    HVX_Vector acc0 = Q6_V_vzero();
-    HVX_Vector acc1 = Q6_V_vzero();
-    HVX_Vector acc2 = Q6_V_vzero();
-    HVX_Vector acc3 = Q6_V_vzero();
-    const HVX_Vector vmul = hvx_vec_splat_f32(mul);
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-
-        HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vmul);
-        HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vmul);
-        HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + i * epv), vmul);
-        HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + i * epv), vmul);
-
-        hvx_vmemu(dst0 + i * epv) = out0;
-        hvx_vmemu(dst1 + i * epv) = out1;
-        hvx_vmemu(dst2 + i * epv) = out2;
-        hvx_vmemu(dst3 + i * epv) = out3;
-
-        acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot));
-        acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot));
-        acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot));
-        acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector zero = Q6_V_vzero();
-
-        HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + off), vmul);
-        HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + off), vmul);
-        HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + off), vmul);
-        HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + off), vmul);
-
-        hvx_vec_store_u(dst0 + off, tail * sizeof(float), out0);
-        hvx_vec_store_u(dst1 + off, tail * sizeof(float), out1);
-        hvx_vec_store_u(dst2 + off, tail * sizeof(float), out2);
-        hvx_vec_store_u(dst3 + off, tail * sizeof(float), out3);
-
-        acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero));
-        acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero));
-        acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero));
-        acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero));
-    }
-
-    HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } };
-    hvx_vec_store_u(sums, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(acc));
-}
-
-static inline void gdn_add_scaled_dot4_f32(float * restrict dst0, float * restrict dst1,
-        float * restrict dst2, float * restrict dst3, const float * restrict src,
-        const float * restrict scale, const float * restrict dot, uint32_t n,
-        float * restrict sums) {
-    HVX_Vector acc0 = Q6_V_vzero();
-    HVX_Vector acc1 = Q6_V_vzero();
-    HVX_Vector acc2 = Q6_V_vzero();
-    HVX_Vector acc3 = Q6_V_vzero();
-    const HVX_Vector scale0 = hvx_vec_splat_f32(scale[0]);
-    const HVX_Vector scale1 = hvx_vec_splat_f32(scale[1]);
-    const HVX_Vector scale2 = hvx_vec_splat_f32(scale[2]);
-    const HVX_Vector scale3 = hvx_vec_splat_f32(scale[3]);
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vs = hvx_vmem(src + i * epv);
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-
-        HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + i * epv), hvx_vec_mul_f32_f32(vs, scale0));
-        HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + i * epv), hvx_vec_mul_f32_f32(vs, scale1));
-        HVX_Vector out2 = hvx_vec_add_f32_f32(hvx_vmemu(dst2 + i * epv), hvx_vec_mul_f32_f32(vs, scale2));
-        HVX_Vector out3 = hvx_vec_add_f32_f32(hvx_vmemu(dst3 + i * epv), hvx_vec_mul_f32_f32(vs, scale3));
-
-        hvx_vmemu(dst0 + i * epv) = out0;
-        hvx_vmemu(dst1 + i * epv) = out1;
-        hvx_vmemu(dst2 + i * epv) = out2;
-        hvx_vmemu(dst3 + i * epv) = out3;
-
-        acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot));
-        acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot));
-        acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot));
-        acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vs = hvx_vmem(src + off);
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector zero = Q6_V_vzero();
-
-        HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + off), hvx_vec_mul_f32_f32(vs, scale0));
-        HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + off), hvx_vec_mul_f32_f32(vs, scale1));
-        HVX_Vector out2 = hvx_vec_add_f32_f32(hvx_vmemu(dst2 + off), hvx_vec_mul_f32_f32(vs, scale2));
-        HVX_Vector out3 = hvx_vec_add_f32_f32(hvx_vmemu(dst3 + off), hvx_vec_mul_f32_f32(vs, scale3));
-
-        hvx_vec_store_u(dst0 + off, tail * sizeof(float), out0);
-        hvx_vec_store_u(dst1 + off, tail * sizeof(float), out1);
-        hvx_vec_store_u(dst2 + off, tail * sizeof(float), out2);
-        hvx_vec_store_u(dst3 + off, tail * sizeof(float), out3);
-
-        acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero));
-        acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero));
-        acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero));
-        acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero));
-    }
-
-    HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } };
-    hvx_vec_store_u(sums, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(acc));
-}
-
-static inline void gdn_mul_dot8_f32(float * restrict dst0, float * restrict dst1,
-        float * restrict dst2, float * restrict dst3, float * restrict dst4,
-        float * restrict dst5, float * restrict dst6, float * restrict dst7,
-        const float * restrict mul, const float * restrict dot, uint32_t n,
-        float * restrict sums) {
-    HVX_Vector acc0 = Q6_V_vzero();
-    HVX_Vector acc1 = Q6_V_vzero();
-    HVX_Vector acc2 = Q6_V_vzero();
-    HVX_Vector acc3 = Q6_V_vzero();
-    HVX_Vector acc4 = Q6_V_vzero();
-    HVX_Vector acc5 = Q6_V_vzero();
-    HVX_Vector acc6 = Q6_V_vzero();
-    HVX_Vector acc7 = Q6_V_vzero();
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vm = hvx_vmem(mul + i * epv);
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-
-        HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vm);
-        HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vm);
-        HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + i * epv), vm);
-        HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + i * epv), vm);
-        HVX_Vector out4 = hvx_vec_mul_f32_f32(hvx_vmemu(dst4 + i * epv), vm);
-        HVX_Vector out5 = hvx_vec_mul_f32_f32(hvx_vmemu(dst5 + i * epv), vm);
-        HVX_Vector out6 = hvx_vec_mul_f32_f32(hvx_vmemu(dst6 + i * epv), vm);
-        HVX_Vector out7 = hvx_vec_mul_f32_f32(hvx_vmemu(dst7 + i * epv), vm);
-
-        hvx_vmemu(dst0 + i * epv) = out0;
-        hvx_vmemu(dst1 + i * epv) = out1;
-        hvx_vmemu(dst2 + i * epv) = out2;
-        hvx_vmemu(dst3 + i * epv) = out3;
-        hvx_vmemu(dst4 + i * epv) = out4;
-        hvx_vmemu(dst5 + i * epv) = out5;
-        hvx_vmemu(dst6 + i * epv) = out6;
-        hvx_vmemu(dst7 + i * epv) = out7;
-
-        acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot));
-        acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot));
-        acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot));
-        acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot));
-        acc4 = hvx_vec_add_f32_f32(acc4, hvx_vec_mul_f32_f32(out4, vdot));
-        acc5 = hvx_vec_add_f32_f32(acc5, hvx_vec_mul_f32_f32(out5, vdot));
-        acc6 = hvx_vec_add_f32_f32(acc6, hvx_vec_mul_f32_f32(out6, vdot));
-        acc7 = hvx_vec_add_f32_f32(acc7, hvx_vec_mul_f32_f32(out7, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vm = hvx_vmem(mul + off);
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector zero = Q6_V_vzero();
-
-        HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + off), vm);
-        HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + off), vm);
-        HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + off), vm);
-        HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + off), vm);
-        HVX_Vector out4 = hvx_vec_mul_f32_f32(hvx_vmemu(dst4 + off), vm);
-        HVX_Vector out5 = hvx_vec_mul_f32_f32(hvx_vmemu(dst5 + off), vm);
-        HVX_Vector out6 = hvx_vec_mul_f32_f32(hvx_vmemu(dst6 + off), vm);
-        HVX_Vector out7 = hvx_vec_mul_f32_f32(hvx_vmemu(dst7 + off), vm);
-
-        hvx_vec_store_u(dst0 + off, tail * sizeof(float), out0);
-        hvx_vec_store_u(dst1 + off, tail * sizeof(float), out1);
-        hvx_vec_store_u(dst2 + off, tail * sizeof(float), out2);
-        hvx_vec_store_u(dst3 + off, tail * sizeof(float), out3);
-        hvx_vec_store_u(dst4 + off, tail * sizeof(float), out4);
-        hvx_vec_store_u(dst5 + off, tail * sizeof(float), out5);
-        hvx_vec_store_u(dst6 + off, tail * sizeof(float), out6);
-        hvx_vec_store_u(dst7 + off, tail * sizeof(float), out7);
-
-        acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero));
-        acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero));
-        acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero));
-        acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero));
-        acc4 = hvx_vec_add_f32_f32(acc4, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out4, vdot), zero));
-        acc5 = hvx_vec_add_f32_f32(acc5, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out5, vdot), zero));
-        acc6 = hvx_vec_add_f32_f32(acc6, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out6, vdot), zero));
-        acc7 = hvx_vec_add_f32_f32(acc7, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out7, vdot), zero));
-    }
-
-    HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } };
-    HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } };
-    hvx_vec_store_u(sums + 0, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accA));
-    hvx_vec_store_u(sums + 4, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accB));
-}
-
-static inline void gdn_mul_scalar_dot8_f32(float * restrict dst0, float * restrict dst1,
-        float * restrict dst2, float * restrict dst3, float * restrict dst4,
-        float * restrict dst5, float * restrict dst6, float * restrict dst7,
-        float mul, const float * restrict dot, uint32_t n, float * restrict sums) {
-    HVX_Vector acc0 = Q6_V_vzero();
-    HVX_Vector acc1 = Q6_V_vzero();
-    HVX_Vector acc2 = Q6_V_vzero();
-    HVX_Vector acc3 = Q6_V_vzero();
-    HVX_Vector acc4 = Q6_V_vzero();
-    HVX_Vector acc5 = Q6_V_vzero();
-    HVX_Vector acc6 = Q6_V_vzero();
-    HVX_Vector acc7 = Q6_V_vzero();
-    const HVX_Vector vmul = hvx_vec_splat_f32(mul);
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-
-        HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vmul);
-        HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vmul);
-        HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + i * epv), vmul);
-        HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + i * epv), vmul);
-        HVX_Vector out4 = hvx_vec_mul_f32_f32(hvx_vmemu(dst4 + i * epv), vmul);
-        HVX_Vector out5 = hvx_vec_mul_f32_f32(hvx_vmemu(dst5 + i * epv), vmul);
-        HVX_Vector out6 = hvx_vec_mul_f32_f32(hvx_vmemu(dst6 + i * epv), vmul);
-        HVX_Vector out7 = hvx_vec_mul_f32_f32(hvx_vmemu(dst7 + i * epv), vmul);
-
-        hvx_vmemu(dst0 + i * epv) = out0;
-        hvx_vmemu(dst1 + i * epv) = out1;
-        hvx_vmemu(dst2 + i * epv) = out2;
-        hvx_vmemu(dst3 + i * epv) = out3;
-        hvx_vmemu(dst4 + i * epv) = out4;
-        hvx_vmemu(dst5 + i * epv) = out5;
-        hvx_vmemu(dst6 + i * epv) = out6;
-        hvx_vmemu(dst7 + i * epv) = out7;
-
-        acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot));
-        acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot));
-        acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot));
-        acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot));
-        acc4 = hvx_vec_add_f32_f32(acc4, hvx_vec_mul_f32_f32(out4, vdot));
-        acc5 = hvx_vec_add_f32_f32(acc5, hvx_vec_mul_f32_f32(out5, vdot));
-        acc6 = hvx_vec_add_f32_f32(acc6, hvx_vec_mul_f32_f32(out6, vdot));
-        acc7 = hvx_vec_add_f32_f32(acc7, hvx_vec_mul_f32_f32(out7, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector zero = Q6_V_vzero();
-
-        HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + off), vmul);
-        HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + off), vmul);
-        HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + off), vmul);
-        HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + off), vmul);
-        HVX_Vector out4 = hvx_vec_mul_f32_f32(hvx_vmemu(dst4 + off), vmul);
-        HVX_Vector out5 = hvx_vec_mul_f32_f32(hvx_vmemu(dst5 + off), vmul);
-        HVX_Vector out6 = hvx_vec_mul_f32_f32(hvx_vmemu(dst6 + off), vmul);
-        HVX_Vector out7 = hvx_vec_mul_f32_f32(hvx_vmemu(dst7 + off), vmul);
-
-        hvx_vec_store_u(dst0 + off, tail * sizeof(float), out0);
-        hvx_vec_store_u(dst1 + off, tail * sizeof(float), out1);
-        hvx_vec_store_u(dst2 + off, tail * sizeof(float), out2);
-        hvx_vec_store_u(dst3 + off, tail * sizeof(float), out3);
-        hvx_vec_store_u(dst4 + off, tail * sizeof(float), out4);
-        hvx_vec_store_u(dst5 + off, tail * sizeof(float), out5);
-        hvx_vec_store_u(dst6 + off, tail * sizeof(float), out6);
-        hvx_vec_store_u(dst7 + off, tail * sizeof(float), out7);
-
-        acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero));
-        acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero));
-        acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero));
-        acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero));
-        acc4 = hvx_vec_add_f32_f32(acc4, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out4, vdot), zero));
-        acc5 = hvx_vec_add_f32_f32(acc5, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out5, vdot), zero));
-        acc6 = hvx_vec_add_f32_f32(acc6, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out6, vdot), zero));
-        acc7 = hvx_vec_add_f32_f32(acc7, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out7, vdot), zero));
-    }
-
-    HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } };
-    HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } };
-    hvx_vec_store_u(sums + 0, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accA));
-    hvx_vec_store_u(sums + 4, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accB));
-}
-
-static inline void gdn_add_scaled_dot8_f32(float * restrict dst0, float * restrict dst1,
-        float * restrict dst2, float * restrict dst3, float * restrict dst4,
-        float * restrict dst5, float * restrict dst6, float * restrict dst7,
-        const float * restrict src, const float * restrict scale,
-        const float * restrict dot, uint32_t n, float * restrict sums) {
-    HVX_Vector acc0 = Q6_V_vzero();
-    HVX_Vector acc1 = Q6_V_vzero();
-    HVX_Vector acc2 = Q6_V_vzero();
-    HVX_Vector acc3 = Q6_V_vzero();
-    HVX_Vector acc4 = Q6_V_vzero();
-    HVX_Vector acc5 = Q6_V_vzero();
-    HVX_Vector acc6 = Q6_V_vzero();
-    HVX_Vector acc7 = Q6_V_vzero();
-    const HVX_Vector scale0 = hvx_vec_splat_f32(scale[0]);
-    const HVX_Vector scale1 = hvx_vec_splat_f32(scale[1]);
-    const HVX_Vector scale2 = hvx_vec_splat_f32(scale[2]);
-    const HVX_Vector scale3 = hvx_vec_splat_f32(scale[3]);
-    const HVX_Vector scale4 = hvx_vec_splat_f32(scale[4]);
-    const HVX_Vector scale5 = hvx_vec_splat_f32(scale[5]);
-    const HVX_Vector scale6 = hvx_vec_splat_f32(scale[6]);
-    const HVX_Vector scale7 = hvx_vec_splat_f32(scale[7]);
-
-    const uint32_t epv = 128 / sizeof(float);
-    const uint32_t nvec = n / epv;
-    const uint32_t tail = n % epv;
-    for (uint32_t i = 0; i < nvec; ++i) {
-        HVX_Vector vs = hvx_vmem(src + i * epv);
-        HVX_Vector vdot = hvx_vmem(dot + i * epv);
-
-        HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + i * epv), hvx_vec_mul_f32_f32(vs, scale0));
-        HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + i * epv), hvx_vec_mul_f32_f32(vs, scale1));
-        HVX_Vector out2 = hvx_vec_add_f32_f32(hvx_vmemu(dst2 + i * epv), hvx_vec_mul_f32_f32(vs, scale2));
-        HVX_Vector out3 = hvx_vec_add_f32_f32(hvx_vmemu(dst3 + i * epv), hvx_vec_mul_f32_f32(vs, scale3));
-        HVX_Vector out4 = hvx_vec_add_f32_f32(hvx_vmemu(dst4 + i * epv), hvx_vec_mul_f32_f32(vs, scale4));
-        HVX_Vector out5 = hvx_vec_add_f32_f32(hvx_vmemu(dst5 + i * epv), hvx_vec_mul_f32_f32(vs, scale5));
-        HVX_Vector out6 = hvx_vec_add_f32_f32(hvx_vmemu(dst6 + i * epv), hvx_vec_mul_f32_f32(vs, scale6));
-        HVX_Vector out7 = hvx_vec_add_f32_f32(hvx_vmemu(dst7 + i * epv), hvx_vec_mul_f32_f32(vs, scale7));
-
-        hvx_vmemu(dst0 + i * epv) = out0;
-        hvx_vmemu(dst1 + i * epv) = out1;
-        hvx_vmemu(dst2 + i * epv) = out2;
-        hvx_vmemu(dst3 + i * epv) = out3;
-        hvx_vmemu(dst4 + i * epv) = out4;
-        hvx_vmemu(dst5 + i * epv) = out5;
-        hvx_vmemu(dst6 + i * epv) = out6;
-        hvx_vmemu(dst7 + i * epv) = out7;
-
-        acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot));
-        acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot));
-        acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot));
-        acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot));
-        acc4 = hvx_vec_add_f32_f32(acc4, hvx_vec_mul_f32_f32(out4, vdot));
-        acc5 = hvx_vec_add_f32_f32(acc5, hvx_vec_mul_f32_f32(out5, vdot));
-        acc6 = hvx_vec_add_f32_f32(acc6, hvx_vec_mul_f32_f32(out6, vdot));
-        acc7 = hvx_vec_add_f32_f32(acc7, hvx_vec_mul_f32_f32(out7, vdot));
-    }
-
-    if (tail) {
-        const uint32_t off = nvec * epv;
-        HVX_Vector vs = hvx_vmem(src + off);
-        HVX_Vector vdot = hvx_vmem(dot + off);
-        HVX_VectorPred mask = Q6_Q_vsetq2_R(tail * sizeof(float));
-        HVX_Vector zero = Q6_V_vzero();
-
-        HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + off), hvx_vec_mul_f32_f32(vs, scale0));
-        HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + off), hvx_vec_mul_f32_f32(vs, scale1));
-        HVX_Vector out2 = hvx_vec_add_f32_f32(hvx_vmemu(dst2 + off), hvx_vec_mul_f32_f32(vs, scale2));
-        HVX_Vector out3 = hvx_vec_add_f32_f32(hvx_vmemu(dst3 + off), hvx_vec_mul_f32_f32(vs, scale3));
-        HVX_Vector out4 = hvx_vec_add_f32_f32(hvx_vmemu(dst4 + off), hvx_vec_mul_f32_f32(vs, scale4));
-        HVX_Vector out5 = hvx_vec_add_f32_f32(hvx_vmemu(dst5 + off), hvx_vec_mul_f32_f32(vs, scale5));
-        HVX_Vector out6 = hvx_vec_add_f32_f32(hvx_vmemu(dst6 + off), hvx_vec_mul_f32_f32(vs, scale6));
-        HVX_Vector out7 = hvx_vec_add_f32_f32(hvx_vmemu(dst7 + off), hvx_vec_mul_f32_f32(vs, scale7));
-
-        hvx_vec_store_u(dst0 + off, tail * sizeof(float), out0);
-        hvx_vec_store_u(dst1 + off, tail * sizeof(float), out1);
-        hvx_vec_store_u(dst2 + off, tail * sizeof(float), out2);
-        hvx_vec_store_u(dst3 + off, tail * sizeof(float), out3);
-        hvx_vec_store_u(dst4 + off, tail * sizeof(float), out4);
-        hvx_vec_store_u(dst5 + off, tail * sizeof(float), out5);
-        hvx_vec_store_u(dst6 + off, tail * sizeof(float), out6);
-        hvx_vec_store_u(dst7 + off, tail * sizeof(float), out7);
-
-        acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero));
-        acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero));
-        acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero));
-        acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero));
-        acc4 = hvx_vec_add_f32_f32(acc4, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out4, vdot), zero));
-        acc5 = hvx_vec_add_f32_f32(acc5, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out5, vdot), zero));
-        acc6 = hvx_vec_add_f32_f32(acc6, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out6, vdot), zero));
-        acc7 = hvx_vec_add_f32_f32(acc7, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out7, vdot), zero));
-    }
-
-    HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } };
-    HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } };
-    hvx_vec_store_u(sums + 0, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accA));
-    hvx_vec_store_u(sums + 4, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accB));
-}
-
-static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, void * data) {
-    struct htp_gdn_context * gctx = (struct htp_gdn_context *) data;
-    struct htp_ops_context * octx = gctx->octx;
-
-    const struct htp_tensor * q     = octx->src[0];
-    const struct htp_tensor * k     = octx->src[1];
-    const struct htp_tensor * v     = octx->src[2];
-    const struct htp_tensor * g     = octx->src[3];
-    const struct htp_tensor * beta  = octx->src[4];
-    const struct htp_tensor * state = octx->src[5];
-    const struct htp_tensor * dst   = octx->dst;
-
-    const uint32_t S_v      = v->ne[0];
-    const uint32_t H        = v->ne[1];
-    const uint32_t n_tokens = v->ne[2];
-    const uint32_t n_seqs   = v->ne[3];
-
-    const uint32_t total_rows = H * n_seqs;
-    if (ith >= total_rows) {
-        return;
-    }
-
-    const uint32_t rq3 = n_seqs / q->ne[3];
-    const uint32_t rk3 = n_seqs / k->ne[3];
-    const float scale = 1.0f / sqrtf((float) S_v);
-
-    float * dst_base       = (float *) (uintptr_t) dst->data;
-    float * state_out_base = dst_base + (uint64_t) S_v * H * n_tokens * n_seqs;
-    const float * state_in_base = (const float *) (uintptr_t) state->data;
-
-    const bool kda = (g->ne[0] == S_v);
-    float local_gate[HTP_GDN_MAX_SV] __attribute__((aligned(128)));
-    float local_q[HTP_GDN_MAX_SV] __attribute__((aligned(128)));
-    float local_k[HTP_GDN_MAX_SV] __attribute__((aligned(128)));
-    float local_sums[4] __attribute__((aligned(128)));
-
-    for (uint32_t ir = ith; ir < total_rows; ir += nth) {
-        const uint32_t iv1 = ir % H;
-        const uint32_t iv3 = ir / H;
-
-        const uint32_t iq1 = iv1 % q->ne[1];
-        const uint32_t ik1 = iv1 % k->ne[1];
-        const uint32_t iq3 = iv3 / rq3;
-        const uint32_t ik3 = iv3 / rk3;
-
-        float * s_out = state_out_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v;
-        const float * s_in = state_in_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v;
-
-        memcpy(s_out, s_in, gctx->state_bytes);
-        float * s_work = s_out;
-
-        float * attn_data = dst_base + ((uint64_t) iv3 * n_tokens * H + iv1) * S_v;
-
-        for (uint32_t t = 0; t < n_tokens; ++t) {
-            const float * q_t = (const float *) ((const uint8_t *) (uintptr_t) q->data +
-                    (uint64_t) iq3 * q->nb[3] + (uint64_t) t * q->nb[2] + (uint64_t) iq1 * q->nb[1]);
-            const float * k_t = (const float *) ((const uint8_t *) (uintptr_t) k->data +
-                    (uint64_t) ik3 * k->nb[3] + (uint64_t) t * k->nb[2] + (uint64_t) ik1 * k->nb[1]);
-            const float * v_t = (const float *) ((const uint8_t *) (uintptr_t) v->data +
-                    (uint64_t) iv3 * v->nb[3] + (uint64_t) t * v->nb[2] + (uint64_t) iv1 * v->nb[1]);
-            const float * g_t = (const float *) ((const uint8_t *) (uintptr_t) g->data +
-                    (uint64_t) iv3 * g->nb[3] + (uint64_t) t * g->nb[2] + (uint64_t) iv1 * g->nb[1]);
-            const float beta_val = *(const float *) ((const uint8_t *) (uintptr_t) beta->data +
-                    (uint64_t) iv3 * beta->nb[3] + (uint64_t) t * beta->nb[2] + (uint64_t) iv1 * beta->nb[1]);
-
-            memcpy(local_q, q_t, (size_t) S_v * sizeof(float));
-            memcpy(local_k, k_t, (size_t) S_v * sizeof(float));
-
-            if (kda) {
-                hvx_exp_f32((uint8_t *) local_gate, (const uint8_t *) g_t, S_v, false);
-
-                uint32_t j = 0;
-                for (; j + 4 <= S_v; j += 4) {
-                    float * row0 = s_work + (uint64_t) (j + 0) * S_v;
-                    float * row1 = s_work + (uint64_t) (j + 1) * S_v;
-                    float * row2 = s_work + (uint64_t) (j + 2) * S_v;
-                    float * row3 = s_work + (uint64_t) (j + 3) * S_v;
-                    gdn_mul_dot4_f32(row0, row1, row2, row3, local_gate, local_k, S_v, local_sums);
-                    float local_delta_b[4] __attribute__((aligned(128)));
-                    for (uint32_t r = 0; r < 4; ++r) {
-                        local_delta_b[r] = (v_t[j + r] - local_sums[r]) * beta_val;
-                    }
-                    gdn_add_scaled_dot4_f32(row0, row1, row2, row3, local_k, local_delta_b, local_q, S_v, local_sums);
-                    for (uint32_t r = 0; r < 4; ++r) {
-                        attn_data[j + r] = local_sums[r] * scale;
-                    }
-                }
-                for (; j < S_v; ++j) {
-                    float * row = s_work + (uint64_t) j * S_v;
-                    const float sum = gdn_mul_dot_f32(row, local_gate, local_k, S_v);
-                    const float dj = (v_t[j] - sum) * beta_val;
-                    attn_data[j] = gdn_add_scaled_dot_f32(row, local_k, dj, local_q, S_v) * scale;
-                }
-            } else {
-                const float gate = expf(g_t[0]);
-                uint32_t j = 0;
-                for (; j + 4 <= S_v; j += 4) {
-                    float * row0 = s_work + (uint64_t) (j + 0) * S_v;
-                    float * row1 = s_work + (uint64_t) (j + 1) * S_v;
-                    float * row2 = s_work + (uint64_t) (j + 2) * S_v;
-                    float * row3 = s_work + (uint64_t) (j + 3) * S_v;
-                    gdn_mul_scalar_dot4_f32(row0, row1, row2, row3, gate, local_k, S_v, local_sums);
-                    float local_delta_b[4] __attribute__((aligned(128)));
-                    for (uint32_t r = 0; r < 4; ++r) {
-                        local_delta_b[r] = (v_t[j + r] - local_sums[r]) * beta_val;
-                    }
-                    gdn_add_scaled_dot4_f32(row0, row1, row2, row3, local_k, local_delta_b, local_q, S_v, local_sums);
-                    for (uint32_t r = 0; r < 4; ++r) {
-                        attn_data[j + r] = local_sums[r] * scale;
-                    }
-                }
-                for (; j < S_v; ++j) {
-                    float * row = s_work + (uint64_t) j * S_v;
-                    const float sum = gdn_mul_scalar_dot_f32(row, gate, local_k, S_v);
-                    const float dj = (v_t[j] - sum) * beta_val;
-                    attn_data[j] = gdn_add_scaled_dot_f32(row, local_k, dj, local_q, S_v) * scale;
-                }
-            }
-
-            attn_data += (uint64_t) S_v * H;
-        }
-    }
-}
-
-static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, void * data) {
-    struct htp_gdn_context * gctx = (struct htp_gdn_context *) data;
-    struct htp_ops_context * octx = gctx->octx;
-
-    const struct htp_tensor * q     = octx->src[0];
-    const struct htp_tensor * k     = octx->src[1];
-    const struct htp_tensor * v     = octx->src[2];
-    const struct htp_tensor * g     = octx->src[3];
-    const struct htp_tensor * beta  = octx->src[4];
-    const struct htp_tensor * state = octx->src[5];
-    const struct htp_tensor * dst   = octx->dst;
-
-    const uint32_t S_v      = v->ne[0];
-    const uint32_t H        = v->ne[1];
-    const uint32_t n_seqs   = v->ne[3];
-
-    const uint32_t total_rows = H * n_seqs;
-    if (ith >= total_rows) {
-        return;
-    }
-
-    const uint32_t rq3 = n_seqs / q->ne[3];
-    const uint32_t rk3 = n_seqs / k->ne[3];
-    const float scale = 1.0f / sqrtf((float) S_v);
-
-    float * dst_base       = (float *) (uintptr_t) dst->data;
-    float * state_out_base = dst_base + (uint64_t) S_v * H * n_seqs;
-    const float * state_in_base = (const float *) (uintptr_t) state->data;
-
-    const bool kda = (g->ne[0] == S_v);
-    float local_gate[HTP_GDN_MAX_SV] __attribute__((aligned(128)));
-    float local_q[HTP_GDN_MAX_SV] __attribute__((aligned(128)));
-    float local_k[HTP_GDN_MAX_SV] __attribute__((aligned(128)));
-    float local_sums[8] __attribute__((aligned(128)));
-
-    dma_queue * dma = octx->ctx->dma[ith];
-
-    uint8_t * spad = NULL;
-    if (gctx->use_vtcm) {
-        spad = gctx->vtcm_state_base + gctx->vtcm_state_per_thread * ith;
-    }
-
-    for (uint32_t ir = ith; ir < total_rows; ir += nth) {
-        const uint32_t iv1 = ir % H;
-        const uint32_t iv3 = ir / H;
-
-        const uint32_t iq1 = iv1 % q->ne[1];
-        const uint32_t ik1 = iv1 % k->ne[1];
-        const uint32_t iq3 = iv3 / rq3;
-        const uint32_t ik3 = iv3 / rk3;
-
-        float * s_out = state_out_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v;
-        const float * s_in = state_in_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v;
-        float * s_work;
-
-        if (spad) {
-            dma_queue_push(dma, dma_make_ptr(spad, s_in),
-                           S_v * sizeof(float), S_v * sizeof(float),
-                           S_v * sizeof(float), S_v);
-            dma_queue_pop(dma);
-            s_work = (float *) spad;
-        } else {
-            s_work = s_out;
-            memcpy(s_work, s_in, gctx->state_bytes);
-        }
-
-        float * attn_data = dst_base + ((uint64_t) iv3 * H + iv1) * S_v;
-
-        const float * q_t = (const float *) ((const uint8_t *) (uintptr_t) q->data +
-                (uint64_t) iq3 * q->nb[3] + (uint64_t) iq1 * q->nb[1]);
-        const float * k_t = (const float *) ((const uint8_t *) (uintptr_t) k->data +
-                (uint64_t) ik3 * k->nb[3] + (uint64_t) ik1 * k->nb[1]);
-        const float * v_t = (const float *) ((const uint8_t *) (uintptr_t) v->data +
-                (uint64_t) iv3 * v->nb[3] + (uint64_t) iv1 * v->nb[1]);
-        const float * g_t = (const float *) ((const uint8_t *) (uintptr_t) g->data +
-                (uint64_t) iv3 * g->nb[3] + (uint64_t) iv1 * g->nb[1]);
-        const float beta_val = *(const float *) ((const uint8_t *) (uintptr_t) beta->data +
-                (uint64_t) iv3 * beta->nb[3] + (uint64_t) iv1 * beta->nb[1]);
-
-        memcpy(local_q, q_t, (size_t) S_v * sizeof(float));
-        memcpy(local_k, k_t, (size_t) S_v * sizeof(float));
-
-        if (kda) {
-            hvx_exp_f32((uint8_t *) local_gate, (const uint8_t *) g_t, S_v, false);
-
-            uint32_t j = 0;
-            for (; j + 8 <= S_v; j += 8) {
-                float * row0 = s_work + (uint64_t) (j + 0) * S_v;
-                float * row1 = s_work + (uint64_t) (j + 1) * S_v;
-                float * row2 = s_work + (uint64_t) (j + 2) * S_v;
-                float * row3 = s_work + (uint64_t) (j + 3) * S_v;
-                float * row4 = s_work + (uint64_t) (j + 4) * S_v;
-                float * row5 = s_work + (uint64_t) (j + 5) * S_v;
-                float * row6 = s_work + (uint64_t) (j + 6) * S_v;
-                float * row7 = s_work + (uint64_t) (j + 7) * S_v;
-                gdn_mul_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7,
-                                 local_gate, local_k, S_v, local_sums);
-                float local_delta_b[8] __attribute__((aligned(128)));
-                for (uint32_t r = 0; r < 8; ++r) {
-                    local_delta_b[r] = (v_t[j + r] - local_sums[r]) * beta_val;
-                }
-                gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7,
-                                        local_k, local_delta_b, local_q, S_v, local_sums);
-                for (uint32_t r = 0; r < 8; ++r) {
-                    attn_data[j + r] = local_sums[r] * scale;
-                }
-            }
-            for (; j + 4 <= S_v; j += 4) {
-                float * row0 = s_work + (uint64_t) (j + 0) * S_v;
-                float * row1 = s_work + (uint64_t) (j + 1) * S_v;
-                float * row2 = s_work + (uint64_t) (j + 2) * S_v;
-                float * row3 = s_work + (uint64_t) (j + 3) * S_v;
-                gdn_mul_dot4_f32(row0, row1, row2, row3, local_gate, local_k, S_v, local_sums);
-                float local_delta_b[4] __attribute__((aligned(128)));
-                for (uint32_t r = 0; r < 4; ++r) {
-                    local_delta_b[r] = (v_t[j + r] - local_sums[r]) * beta_val;
-                }
-                gdn_add_scaled_dot4_f32(row0, row1, row2, row3, local_k, local_delta_b, local_q, S_v, local_sums);
-                for (uint32_t r = 0; r < 4; ++r) {
-                    attn_data[j + r] = local_sums[r] * scale;
-                }
-            }
-            for (; j < S_v; ++j) {
-                float * row = s_work + (uint64_t) j * S_v;
-                const float sum = gdn_mul_dot_f32(row, local_gate, local_k, S_v);
-                const float dj = (v_t[j] - sum) * beta_val;
-                attn_data[j] = gdn_add_scaled_dot_f32(row, local_k, dj, local_q, S_v) * scale;
-            }
-        } else {
-            const float gate = expf(g_t[0]);
-            uint32_t j = 0;
-            for (; j + 8 <= S_v; j += 8) {
-                float * row0 = s_work + (uint64_t) (j + 0) * S_v;
-                float * row1 = s_work + (uint64_t) (j + 1) * S_v;
-                float * row2 = s_work + (uint64_t) (j + 2) * S_v;
-                float * row3 = s_work + (uint64_t) (j + 3) * S_v;
-                float * row4 = s_work + (uint64_t) (j + 4) * S_v;
-                float * row5 = s_work + (uint64_t) (j + 5) * S_v;
-                float * row6 = s_work + (uint64_t) (j + 6) * S_v;
-                float * row7 = s_work + (uint64_t) (j + 7) * S_v;
-                gdn_mul_scalar_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7,
-                                        gate, local_k, S_v, local_sums);
-                float local_delta_b[8] __attribute__((aligned(128)));
-                for (uint32_t r = 0; r < 8; ++r) {
-                    local_delta_b[r] = (v_t[j + r] - local_sums[r]) * beta_val;
-                }
-                gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7,
-                                        local_k, local_delta_b, local_q, S_v, local_sums);
-                for (uint32_t r = 0; r < 8; ++r) {
-                    attn_data[j + r] = local_sums[r] * scale;
-                }
-            }
-            for (; j + 4 <= S_v; j += 4) {
-                float * row0 = s_work + (uint64_t) (j + 0) * S_v;
-                float * row1 = s_work + (uint64_t) (j + 1) * S_v;
-                float * row2 = s_work + (uint64_t) (j + 2) * S_v;
-                float * row3 = s_work + (uint64_t) (j + 3) * S_v;
-                gdn_mul_scalar_dot4_f32(row0, row1, row2, row3, gate, local_k, S_v, local_sums);
-                float local_delta_b[4] __attribute__((aligned(128)));
-                for (uint32_t r = 0; r < 4; ++r) {
-                    local_delta_b[r] = (v_t[j + r] - local_sums[r]) * beta_val;
-                }
-                gdn_add_scaled_dot4_f32(row0, row1, row2, row3, local_k, local_delta_b, local_q, S_v, local_sums);
-                for (uint32_t r = 0; r < 4; ++r) {
-                    attn_data[j + r] = local_sums[r] * scale;
-                }
-            }
-            for (; j < S_v; ++j) {
-                float * row = s_work + (uint64_t) j * S_v;
-                const float sum = gdn_mul_scalar_dot_f32(row, gate, local_k, S_v);
-                const float dj = (v_t[j] - sum) * beta_val;
-                attn_data[j] = gdn_add_scaled_dot_f32(row, local_k, dj, local_q, S_v) * scale;
-            }
-        }
-
-        if (spad) {
-            dma_queue_push(dma, dma_make_ptr(s_out, spad),
-                           S_v * sizeof(float), S_v * sizeof(float),
-                           S_v * sizeof(float), S_v);
-            dma_queue_pop(dma);
-        }
-    }
-}
-
-int op_gated_delta_net(struct htp_ops_context * octx) {
-    const struct htp_tensor * q     = octx->src[0];
-    const struct htp_tensor * k     = octx->src[1];
-    const struct htp_tensor * v     = octx->src[2];
-    const struct htp_tensor * g     = octx->src[3];
-    const struct htp_tensor * beta  = octx->src[4];
-    const struct htp_tensor * state = octx->src[5];
-    const struct htp_tensor * dst   = octx->dst;
-
-    if (!q || !k || !v || !g || !beta || !state || !dst) {
-        return HTP_STATUS_INVAL_PARAMS;
-    }
-
-    if (q->type != HTP_TYPE_F32 || k->type != HTP_TYPE_F32 || v->type != HTP_TYPE_F32 ||
-        g->type != HTP_TYPE_F32 || beta->type != HTP_TYPE_F32 || state->type != HTP_TYPE_F32 ||
-        dst->type != HTP_TYPE_F32) {
-        return HTP_STATUS_NO_SUPPORT;
-    }
-
-    const uint32_t S_v      = v->ne[0];
-    const uint32_t H        = v->ne[1];
-    const uint32_t n_tokens = v->ne[2];
-    const uint32_t n_seqs   = v->ne[3];
-
-    if (S_v == 0 || S_v > HTP_GDN_MAX_SV || H == 0 || n_tokens == 0 || n_seqs == 0) {
-        return HTP_STATUS_NO_SUPPORT;
-    }
-    if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {
-        return HTP_STATUS_NO_SUPPORT;
-    }
-    if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] == 0 || k->ne[1] == 0 ||
-        q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] == 0 || k->ne[3] == 0 ||
-        (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {
-        return HTP_STATUS_NO_SUPPORT;
-    }
-    if (state->ne[0] * state->ne[1] * state->ne[2] * state->ne[3] != S_v * S_v * H * n_seqs) {
-        return HTP_STATUS_NO_SUPPORT;
-    }
-    if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs) {
-        return HTP_STATUS_NO_SUPPORT;
-    }
-
-    if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
-        return HTP_STATUS_OK;
-    }
-
-    struct htp_gdn_context gctx;
-    gctx.octx = octx;
-    gctx.rows_per_thread = (H * n_seqs + octx->n_threads - 1) / octx->n_threads;
-    gctx.state_bytes = (size_t) S_v * S_v * sizeof(float);
-
-    size_t state_aligned = (size_t) S_v * S_v * sizeof(float);
-    state_aligned = (state_aligned + 127) & ~(size_t)127;
-
-    gctx.use_vtcm = false;
-    gctx.vtcm_state_base = NULL;
-    gctx.vtcm_state_per_thread = 0;
-
-    if (n_tokens == 1 && octx->ctx->vtcm_base) {
-        size_t vtcm_total = state_aligned * octx->n_threads;
-        if (octx->ctx->vtcm_size >= vtcm_total) {
-            gctx.use_vtcm = true;
-            gctx.vtcm_state_base = octx->ctx->vtcm_base;
-            gctx.vtcm_state_per_thread = state_aligned;
-        }
-    }
-
-    if (n_tokens == 1) {
-        worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_tg_thread, &gctx, octx->n_threads);
-    } else {
-        worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_pp_thread, &gctx, octx->n_threads);
-    }
-
-    return HTP_STATUS_OK;
-}

ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c

@@ -742,45 +742,17 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst,
 // activations : fp32 -> fp16
 
 static void transfer_activation_chunk_fp32_to_fp16(__fp16 *restrict vtcm_dst, const float *restrict src, int n_rows, int k_block, int k_stride) {
-    const int n_rows_padded = hex_align_up(n_rows, HMX_FP16_TILE_N_ROWS);
-    const int n_rows_tiled  = (n_rows / HMX_FP16_TILE_N_ROWS) * HMX_FP16_TILE_N_ROWS;
-
-    int r = 0;
-
-    #pragma unroll(2)
-    for (r = 0; r < n_rows_tiled; r += 2) {
+    for (int r = 0; r < n_rows; r += 2) {
         int r0 = r / HMX_FP16_TILE_N_ROWS;  // tile row index
         int r1 = r % HMX_FP16_TILE_N_ROWS;  // intra-tile row idx
 
+        const bool next_row_valid = (r + 1) < n_rows;
+
         const HVX_Vector *pv_in0 = (const HVX_Vector *) (src + (r + 0) * k_stride);
         const HVX_Vector *pv_in1 = (const HVX_Vector *) (src + (r + 1) * k_stride);
         for (int c = 0; c < k_block; c += 32) {
             HVX_Vector v0 = *pv_in0++;
-            HVX_Vector v1 = *pv_in1++;
-
-            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
-
-            // compute output position
-            int c0       = c / HMX_FP16_TILE_N_COLS;  // tile column index
-            int tile_idx = r0 * (k_block / HMX_FP16_TILE_N_COLS) + c0;
-
-            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HMX_FP16_TILE_N_ELMS);
-            tile[r1 / 2]     = v_out;
-        }
-    }
-
-    for (; r < n_rows_padded; r += 2) {
-        int r0 = r / HMX_FP16_TILE_N_ROWS;  // tile row index
-        int r1 = r % HMX_FP16_TILE_N_ROWS;  // intra-tile row idx
-
-        const bool row0_valid = r       < n_rows;
-        const bool row1_valid = (r + 1) < n_rows;
-
-        const HVX_Vector *pv_in0 = row0_valid ? (const HVX_Vector *) (src + (r + 0) * k_stride) : NULL;
-        const HVX_Vector *pv_in1 = row1_valid ? (const HVX_Vector *) (src + (r + 1) * k_stride) : NULL;
-        for (int c = 0; c < k_block; c += 32) {
-            HVX_Vector v0 = row0_valid ? *pv_in0++ : Q6_V_vzero();
-            HVX_Vector v1 = row1_valid ? *pv_in1++ : Q6_V_vzero();
+            HVX_Vector v1 = next_row_valid ? *pv_in1++ : Q6_V_vzero();
 
             HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
 
@@ -917,9 +889,7 @@ static __attribute__((noinline)) int mat_mul_qk_0_d16a32_out_stationary(struct h
     // n_block_cost = m*2: each extra N-block re-loads all M×K activation (cheaper).
     const size_t m_block_cost = (size_t) n * 3;
     const size_t n_block_cost = (size_t) m * 2;
-    if (hmx_compute_chunks(vtcm_budget, overhead, per_n, per_m, per_mn,
-                           hex_align_up(m, HMX_FP16_TILE_N_ROWS), n,
-                           m_block_cost, n_block_cost, &M_BLOCK_SIZE,
+    if (hmx_compute_chunks(vtcm_budget, overhead, per_n, per_m, per_mn, m, n, m_block_cost, n_block_cost, &M_BLOCK_SIZE,
                            &N_BLOCK_SIZE, &vtcm_used) != 0) {
         FARF(HIGH, "%s: VTCM too small (m=%d k=%d n=%d budget=%zu)", __func__, m, k, n, vtcm_budget);
         return -1;
@@ -1114,8 +1084,7 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds
 
     if (m >= 128) {
         size_t mc = 0, nc = 0, used = 0;
-        if (hmx_compute_chunks(vtcm_budget, /*overhead=*/256, pipe_per_n, /*per_m=*/vec_dot_size, pipe_per_mn,
-                               hex_align_up(m, HMX_FP16_TILE_N_ROWS), n,
+        if (hmx_compute_chunks(vtcm_budget, /*overhead=*/256, pipe_per_n, /*per_m=*/vec_dot_size, pipe_per_mn, m, n,
                                /*m_block_cost=*/(size_t) n * 3,
                                /*n_block_cost=*/(size_t) m * 2, &mc, &nc, &used) == 0 &&
             hmx_ceil_div((size_t) n, nc) >= 2) {
@@ -1127,8 +1096,7 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds
     }
 
     if (!use_pipeline) {
-        if (hmx_compute_chunks(vtcm_budget, /*overhead=*/256, seq_per_n, /*per_m=*/vec_dot_size, seq_per_mn,
-                               hex_align_up(m, HMX_FP16_TILE_N_ROWS), n,
+        if (hmx_compute_chunks(vtcm_budget, /*overhead=*/256, seq_per_n, /*per_m=*/vec_dot_size, seq_per_mn, m, n,
                                /*m_block_cost=*/(size_t) n * 3,
                                /*n_block_cost=*/(size_t) m * 2, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used) != 0) {
             FARF(HIGH, "%s: VTCM too small (m=%d k=%d n=%d budget=%zu)", __func__, m, k, n, vtcm_budget);
@@ -1464,8 +1432,7 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu
     if (hmx_compute_chunks(vtcm_budget, /*overhead=*/256,
                            /*per_n=*/3 * vec_dot_size,
                            /*per_m=*/group_size * vec_dot_size + f32_scratch_per_m,
-                           /*per_mn=*/sizeof(__fp16),
-                           hex_align_up(params->m, HMX_FP16_TILE_N_ROWS), params->n,
+                           /*per_mn=*/sizeof(__fp16), params->m, params->n,
                            /*m_block_cost=*/(size_t) params->n,
                            /*n_block_cost=*/(size_t) params->m, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used) != 0) {
         FARF(HIGH, "%s: grouped path does not fit VTCM, falling back to legacy batched loop", __func__);
@@ -1645,7 +1612,7 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co
                            /*per_n=*/3 * vec_dot_size,                  // W + S0 + S1
                            /*per_m=*/vec_dot_size + f32_scratch_per_m,  // A + optional F32 scratch
                            /*per_mn=*/sizeof(__fp16),                   // O
-                           hex_align_up(m, HMX_FP16_TILE_N_ROWS), n,
+                           m, n,
                            /*m_block_cost=*/(size_t) n,
                            /*n_block_cost=*/(size_t) m, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used) != 0) {
         FARF(HIGH, "%s: VTCM too small (m=%d k=%d n=%d budget=%zu)", __func__, m, k, n, vtcm_budget);

ggml/src/ggml-hexagon/htp/htp-ctx.h

@@ -106,6 +106,5 @@ int op_cumsum(struct htp_ops_context * octx);
 int op_fill(struct htp_ops_context * octx);
 int op_diag(struct htp_ops_context * octx);
 int op_solve_tri(struct htp_ops_context * octx);
-int op_gated_delta_net(struct htp_ops_context * octx);
 
 #endif /* HTP_CTX_H */

ggml/src/ggml-hexagon/htp/htp-ops.h

@@ -83,9 +83,6 @@ enum htp_op_code {
     HTP_OP_FILL,
     HTP_OP_DIAG,
     HTP_OP_SOLVE_TRI,
-    HTP_OP_L2_NORM,
-    HTP_OP_GATED_DELTA_NET,
-
     HTP_OP_INVALID
 };
 

ggml/src/ggml-hexagon/htp/main.c

@@ -542,7 +542,6 @@ static int execute_op(struct htp_ops_context * octx) {
         case HTP_OP_UNARY_SIGMOID:
         case HTP_OP_UNARY_NEG:
         case HTP_OP_UNARY_EXP:
-        case HTP_OP_L2_NORM:
             return op_unary(octx);
 
         case HTP_OP_UNARY_SILU:
@@ -594,9 +593,6 @@ static int execute_op(struct htp_ops_context * octx) {
         case HTP_OP_SOLVE_TRI:
             return op_solve_tri(octx);
 
-        case HTP_OP_GATED_DELTA_NET:
-            return op_gated_delta_net(octx);
-
         case HTP_OP_INVALID:
             break;
 

ggml/src/ggml-hexagon/htp/matmul-ops.c

@@ -2991,10 +2991,12 @@ int op_matmul(struct htp_ops_context * octx) {
         return op_matmul_hvx(octx);
     }
 
-    // M alignment: Use HMX when M >= 32, the last partial tile (m_total % 32 rows)
-    //  is handled by HMX itself; when M < 32  fall back to HVX.
+    // M alignment: when M > 32 but not 32-aligned, we split into
+    // HMX (first m_hmx = M & ~31 rows) + HVX (remaining m_tail rows).
+    // When M <= 32 and not 32-aligned, fall back entirely to HVX.
     const int m_total = (int) src1->ne[1];
-    const int m_hmx   = m_total & ~31;   // 0 when M < 32
+    const int m_tail  = m_total % 32;
+    const int m_hmx   = m_total - m_tail;
 
     if (m_hmx == 0) {
         return op_matmul_hvx(octx);
@@ -3007,6 +3009,7 @@ int op_matmul(struct htp_ops_context * octx) {
     int k = (int) src0->ne[0];  // inner dimension
     int n = (int) src0->ne[1];  // weight columns
 
+    // --- Phase 1: HMX on the first m_hmx (32-aligned) rows ---
     int ret = -1;
 
     // Row strides in elements. For compact tensors these equal k; for
@@ -3024,7 +3027,7 @@ int op_matmul(struct htp_ops_context * octx) {
                 .dst             = (float *) dst->data,
                 .activation      = (float *) src1->data,
                 .permuted_weight = (const __fp16 *) src0->data,
-                .m               = m_total,
+                .m               = m_hmx,
                 .k               = k,
                 .n               = n,
                 .act_stride      = act_stride,
@@ -3045,12 +3048,12 @@ int op_matmul(struct htp_ops_context * octx) {
         } else {
             ret = hmx_mat_mul_permuted_w16a32(octx->ctx,
                     (float*) dst->data, (float*) src1->data, (const __fp16 *) src0->data,
-                    m_total, k, n, act_stride, wgt_stride);
+                    m_hmx, k, n, act_stride, wgt_stride);
         }
     } else {
         ret = hmx_mat_mul_permuted_qk_0_d16a32(octx->ctx,
                     (float*) dst->data, (float*) src1->data, (const uint8_t *) src0->data,
-                    m_total, k, n, (int) src0->type);
+                    m_hmx, k, n, (int) src0->type);
     }
 
     if (ret != 0) {
@@ -3058,6 +3061,27 @@ int op_matmul(struct htp_ops_context * octx) {
         return op_matmul(octx);
     }
 
+    // --- Phase 2: HVX on the remaining m_tail rows ---
+    if (m_tail > 0) {
+        // copy of src1 and dst
+        struct htp_tensor src1_tail = *src1;
+        struct htp_tensor dst_tail  = *dst;
+
+        src1_tail.ne[1] = m_tail; // only tail rows
+        dst_tail.ne[1]  = m_tail; // only tail rows
+
+        // Offset activation and dst pointers past the HMX-processed rows.
+        // Use nb[1] (row stride in bytes) to compute the byte offset.
+        src1_tail.data += (uint32_t) m_hmx * src1->nb[1];
+        dst_tail.data  += (uint32_t) m_hmx * dst->nb[1];
+
+        octx->src[1] = &src1_tail;
+        octx->dst    = &dst_tail;
+
+        FARF(HIGH, "hmx-matmul: HVX tail m_tail %d src1 %p dst %p", m_tail, (void *) src1_tail.data, (void *) dst_tail.data);
+        return op_matmul_hvx(octx);
+    }
+
     return 0;
 #endif // HTP_HAS_HMX
 }

ggml/src/ggml-hexagon/htp/unary-ops.c

@@ -298,81 +298,6 @@ static void softplus_f32(const float * restrict src,
     }
 }
 
-// --- L2_NORM HVX kernel ---
-// Computes y[i] = x[i] / fmax(sqrt(sum(x[j]^2)), epsilon) for each row.
-// scale = 1/fmax(sqrt(sum), epsilon) is computed entirely in HVX registers
-// using rsqrt + inverse to avoid scalar extraction.
-static void hvx_fast_l2_norm_f32(const uint8_t * restrict src,
-                                 uint8_t * restrict dst,
-                                 uint8_t * restrict pad,
-                                 const int num_elems,
-                                 float     epsilon) {
-    (void)pad;
-
-    const HVX_Vector * restrict v_src = (HVX_Vector *) src;
-    HVX_Vector * restrict v_dst       = (HVX_Vector *) dst;
-
-    HVX_Vector sum_v = hvx_vec_splat_f32(0.0f);
-
-    const int nvec = num_elems / VLEN_FP32;
-    const int nloe = num_elems % VLEN_FP32;
-
-    #pragma unroll(4)
-    for (int i = 0; i < nvec; i++) {
-        HVX_Vector v1 = v_src[i];
-        HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
-        sum_v         = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq);
-    }
-
-    // Include tail elements in the sum-of-squares using a predicate mask
-    if (nloe > 0) {
-        HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
-        HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
-        HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
-        sum_v         = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq);
-    }
-
-    // Compute scale = 1/fmax(sqrt(sum), epsilon) entirely in HVX registers.
-    // hvx_vec_rsqrt_f32 + hvx_vec_inverse_f32 avoids scalar extraction.
-    HVX_Vector sum_sf    = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
-    HVX_Vector rsqrt_v   = hvx_vec_rsqrt_f32(sum_sf);              // 1/sqrt(sum)
-    HVX_Vector sqrt_v    = hvx_vec_inverse_f32(rsqrt_v);            // sqrt(sum)
-    HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
-    HVX_Vector denom_v   = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v);  // fmax(sqrt(sum), epsilon)
-    HVX_Vector scale_v   = hvx_vec_inverse_f32(denom_v);            // 1/fmax(sqrt(sum), epsilon)
-
-    #pragma unroll(4)
-    for (int i = 0; i < nvec; i++) {
-        HVX_Vector v1 = v_src[i];
-        v_dst[i]      = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v));
-    }
-
-    if (nloe > 0) {
-        HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
-        HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
-        HVX_Vector result = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v));
-        hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);
-    }
-}
-
-static void l2_norm_f32(const float * restrict src,
-                        float * restrict dst,
-                        uint8_t * restrict spad,
-                        const uint32_t num_rows,
-                        const uint32_t row_elems,
-                        const size_t   row_size,
-                        int32_t *      op_params) {
-    float epsilon = 0.f;
-    memcpy(&epsilon, op_params, sizeof(float));
-
-    for (uint32_t ir = 0; ir < num_rows; ir++) {
-        const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * row_size));
-        float * restrict dst_f       = (float *)((uint8_t *)dst + (ir * row_size));
-
-        hvx_fast_l2_norm_f32((const uint8_t *)src_f, (uint8_t *)dst_f, spad, row_elems, epsilon);
-    }
-}
-
 static void unary_job_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {
     const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;
     struct htp_ops_context * octx = uctx->octx;
@@ -477,9 +402,6 @@ static void unary_job_f32_per_thread(unsigned int nth, unsigned int ith, void *
             case HTP_OP_UNARY_SOFTPLUS:
                 softplus_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);
                 break;
-            case HTP_OP_L2_NORM:
-                l2_norm_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);
-                break;
             default:
                 break;
         }
@@ -547,9 +469,6 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
         case HTP_OP_UNARY_SOFTPLUS:
             op_type = "softplus-f32";
             break;
-        case HTP_OP_L2_NORM:
-            op_type = "l2norm-f32";
-            break;
 
         default:
             FARF(ERROR, "Unsupported unary Op %u\n", octx->op);

ggml/src/ggml-metal/ggml-metal.cpp

@@ -87,17 +87,17 @@ static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer,
 }
 
 static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {
-    /* .free_buffer   = */ ggml_backend_metal_buffer_shared_free_buffer,
-    /* .get_base      = */ ggml_backend_metal_buffer_shared_get_base,
-    /* .init_tensor   = */ NULL,
-    /* .memset_tensor = */ ggml_backend_metal_buffer_shared_memset_tensor,
-    /* .set_tensor    = */ ggml_backend_metal_buffer_shared_set_tensor,
-    /* .get_tensor    = */ ggml_backend_metal_buffer_shared_get_tensor,
-    /* .set_tensor_2d = */ NULL,
-    /* .get_tensor_2d = */ NULL,
-    /* .cpy_tensor    = */ ggml_backend_metal_buffer_shared_cpy_tensor,
-    /* .clear         = */ ggml_backend_metal_buffer_shared_clear,
-    /* .reset         = */ NULL,
+    /* .free_buffer     = */ ggml_backend_metal_buffer_shared_free_buffer,
+    /* .get_base        = */ ggml_backend_metal_buffer_shared_get_base,
+    /* .init_tensor     = */ NULL,
+    /* .memset_tensor   = */ ggml_backend_metal_buffer_shared_memset_tensor,
+    /* .set_tensor      = */ ggml_backend_metal_buffer_shared_set_tensor,
+    /* .get_tensor      = */ ggml_backend_metal_buffer_shared_get_tensor,
+    /* .set_tensor_2d   = */ NULL,
+    /* .get_tensor_2d   = */ NULL,
+    /* .cpy_tensor      = */ ggml_backend_metal_buffer_shared_cpy_tensor,
+    /* .clear           = */ ggml_backend_metal_buffer_shared_clear,
+    /* .reset           = */ NULL,
 };
 
 // private buffer
@@ -163,17 +163,17 @@ static void ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer
 }
 
 static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {
-    /* .free_buffer   = */ ggml_backend_metal_buffer_private_free_buffer,
-    /* .get_base      = */ ggml_backend_metal_buffer_private_get_base,
-    /* .init_tensor   = */ NULL,
-    /* .memset_tensor = */ ggml_backend_metal_buffer_private_memset_tensor,
-    /* .set_tensor    = */ ggml_backend_metal_buffer_private_set_tensor,
-    /* .get_tensor    = */ ggml_backend_metal_buffer_private_get_tensor,
-    /* .set_tensor_2d = */ NULL,
-    /* .get_tensor_2d = */ NULL,
-    /* .cpy_tensor    = */ ggml_backend_metal_buffer_private_cpy_tensor,
-    /* .clear         = */ ggml_backend_metal_buffer_private_clear,
-    /* .reset         = */ NULL,
+    /* .free_buffer             = */ ggml_backend_metal_buffer_private_free_buffer,
+    /* .get_base                = */ ggml_backend_metal_buffer_private_get_base,
+    /* .init_tensor             = */ NULL,
+    /* .memset_tensor           = */ ggml_backend_metal_buffer_private_memset_tensor,
+    /* .set_tensor              = */ ggml_backend_metal_buffer_private_set_tensor,
+    /* .get_tensor              = */ ggml_backend_metal_buffer_private_get_tensor,
+    /* .set_tensor_2d           = */ NULL,
+    /* .get_tensor_2d           = */ NULL,
+    /* .cpy_tensor              = */ ggml_backend_metal_buffer_private_cpy_tensor,
+    /* .clear                   = */ ggml_backend_metal_buffer_private_clear,
+    /* .reset                   = */ NULL,
 };
 
 static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {

ggml/src/ggml-opencl/CMakeLists.txt

@@ -66,6 +66,8 @@ set(GGML_OPENCL_KERNELS
     diag
     div
     gelu
+    gemv_noshuffle_general
+    gemv_noshuffle
     get_rows
     glu
     group_norm
@@ -73,6 +75,7 @@ set(GGML_OPENCL_KERNELS
     im2col_f32
     im2col_f16
     mean
+    mul_mat_Ab_Bi_8x4
     mul_mv_f16_f16
     mul_mv_f16_f32_1row
     mul_mv_f16_f32_l4
@@ -102,8 +105,6 @@ set(GGML_OPENCL_KERNELS
     mul_mv_id_q8_0_f32_flat
     mul_mv_id_mxfp4_f32
     mul_mv_id_mxfp4_f32_flat
-    gemm_moe_q4_0_f32_ns
-    gemv_moe_q4_0_f32_ns
     gemm_moe_mxfp4_f32
     gemv_moe_mxfp4_f32
     gemm_moe_mxfp4_f32_ns
@@ -119,15 +120,12 @@ set(GGML_OPENCL_KERNELS
     mul_mm_q4_k_f32_l4_lm
     mul_mm_q5_k_f32_l4_lm
     mul_mm_q6_k_f32_l4_lm
-    gemv_noshuffle_q4_0_f32
-    gemv_noshuffle_q4_0_f32_spec
-    gemm_noshuffle_q4_0_f32
+    mul_mm_q8_0_f32_8x4
     gemv_noshuffle_q4_1_f32
     gemm_noshuffle_q4_1_f32
     gemv_noshuffle_iq4_nl_f32
     gemm_noshuffle_iq4_nl_f32
-    gemv_noshuffle_q8_0_f32
-    gemm_noshuffle_q8_0_f32
+    gemv_noshuffle_general_q8_0_f32
     gemv_noshuffle_q4_k_f32
     gemm_noshuffle_q4_k_f32
     gemv_noshuffle_q6_k_f32

ggml/src/ggml-opencl/kernels/cvt.cl

@@ -190,92 +190,6 @@ kernel void kernel_restore_block_q4_0_noshuffle(
     }
 }
 
-kernel void kernel_convert_block_q4_0_trans4_ns(
-    global struct block_q4_0 * src0,
-    __global uint * dst_q,
-    __global half * dst_d,
-    uint ne00,
-    uint ne01
-) {
-    uint i00 = get_global_id(1);
-    uint i01 = get_global_id(0);
-    uint i02 = get_global_id(2);
-
-    uint ne00_blk = ne00 / QK4_0;
-    uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
-    uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
-
-    global struct block_q4_0 * b = src0 + src_blk_offset;
-    dst_d[dst_blk_offset] = b->d;
-
-    // extract quantization and unshuffle
-    ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
-
-    ushort8 post_block = (ushort8)(0);
-
-    uchar * pre_block_ptr = (uchar *)(&pre_block);
-    uchar * post_block_ptr = (uchar *)(&post_block);
-
-    for (int i = 0; i < QK4_0 / 4; ++i) {
-        uchar x0 = pre_block_ptr[2*i + 0];
-        uchar x1 = pre_block_ptr[2*i + 1];
-
-        post_block_ptr[i + 0        ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
-        post_block_ptr[i + QK4_0 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
-    }
-
-    uint4 q_block = as_uint4(post_block);
-
-    uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
-    dst_q[offset] = q_block.x;
-    dst_q[offset + ne01] = q_block.y;
-    dst_q[offset + ne01 * 2] = q_block.z;
-    dst_q[offset + ne01 * 3] = q_block.w;
-}
-
-kernel void kernel_restore_block_q4_0_trans4_ns(
-    __global uint * src_q,
-    __global half * src_d,
-    __global struct block_q4_0 * dst0,
-    uint ne00,
-    uint ne01
-) {
-    uint i00 = get_global_id(1);
-    uint i01 = get_global_id(0);
-    uint i02 = get_global_id(2);
-
-    uint ne00_blk = ne00 / QK4_0;
-    uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
-    uint src_d_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
-
-    __global struct block_q4_0 * b = dst0 + dst_blk_offset;
-    b->d = src_d[src_d_offset];
-
-    // collect transposed quantization parts for a block
-    uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
-    uint4 q_block;
-    q_block.x = src_q[src_q_offset];
-    q_block.y = src_q[src_q_offset + ne01];
-    q_block.z = src_q[src_q_offset + ne01 * 2];
-    q_block.w = src_q[src_q_offset + ne01 * 3];
-
-    ushort8 post_block = as_ushort8(q_block);
-    ushort8 pre_block = (ushort8)(0);
-
-    uchar * pre_block_ptr = (uchar *)(&pre_block);
-    uchar * post_block_ptr = (uchar *)(&post_block);
-
-    for (int i = 0; i < QK4_0 / 4; ++i) {
-        uchar x0 = post_block_ptr[i + 0];
-        uchar x1 = post_block_ptr[i + QK4_0 / 4];
-
-        pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
-        pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
-    }
-
-    ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
-}
-
 //------------------------------------------------------------------------------
 // kernel_convert_block_q4_1
 // Convert the block_q4_1 format to 2 separate arrays (AOS -> SOA).

ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl

@@ -1,252 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable
-#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable
-#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable
-
-#define TILESIZE_K 16
-#define TILESIZE_M 64
-#define TILESIZE_N 32
-
-
-#define dequantize_q4_0(q4, a_f16, scale) \
-    a_f16.s0 = (half)((q4.s0 & 0x000F) - 8) * scale; \
-    a_f16.s1 = (half)(((q4.s0 & 0x00F0) >> 4) - 8) * scale; \
-    a_f16.s2 = (half)(((q4.s0 & 0x0F00) >> 8) - 8) * scale; \
-    a_f16.s3 = (half)(((q4.s0 & 0xF000) >> 12) - 8) * scale; \
-    a_f16.s4 = (half)((q4.s1 & 0x000F) - 8) * scale; \
-    a_f16.s5 = (half)(((q4.s1 & 0x00F0) >> 4) - 8) * scale; \
-    a_f16.s6 = (half)(((q4.s1 & 0x0F00) >> 8) - 8) * scale; \
-    a_f16.s7 = (half)(((q4.s1 & 0xF000) >> 12) - 8) * scale; \
-    a_f16.s8 = (half)((q4.s2 & 0x000F) - 8) * scale; \
-    a_f16.s9 = (half)(((q4.s2 & 0x00F0) >> 4) - 8) * scale; \
-    a_f16.sa = (half)(((q4.s2 & 0x0F00) >> 8) - 8) * scale; \
-    a_f16.sb = (half)(((q4.s2 & 0xF000) >> 12) - 8) * scale; \
-    a_f16.sc = (half)((q4.s3 & 0x000F) - 8) * scale; \
-    a_f16.sd = (half)(((q4.s3 & 0x00F0) >> 4) - 8) * scale; \
-    a_f16.se = (half)(((q4.s3 & 0x0F00) >> 8) - 8) * scale; \
-    a_f16.sf = (half)(((q4.s3 & 0xF000) >> 12) - 8) * scale; \
-
-
-#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \
-    acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \
-    acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \
-    acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \
-    acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \
-    acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \
-    acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \
-    acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \
-    acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \
-    acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \
-    acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \
-    acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \
-    acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \
-    acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \
-    acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \
-    acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \
-    acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \
-    acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \
-    acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \
-    acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \
-    acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \
-    acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \
-    acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \
-    acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \
-    acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \
-    acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \
-    acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \
-    acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \
-    acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \
-    acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \
-    acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \
-    acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \
-    acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \
-    c_reg.lo += convert_float8(acc.lo); \
-    c_reg.hi += convert_float8(acc.hi); \
-    acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \
-    acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \
-    acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \
-    acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \
-    acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \
-    acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \
-    acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \
-    acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \
-    acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \
-    acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \
-    acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \
-    acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \
-    acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \
-    acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \
-    acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \
-    acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \
-    acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \
-    acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \
-    acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \
-    acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \
-    acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \
-    acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \
-    acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \
-    acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \
-    acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \
-    acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \
-    acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \
-    acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \
-    acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \
-    acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \
-    acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \
-    acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \
-    c_reg.lo += convert_float8(acc.lo); \
-    c_reg.hi += convert_float8(acc.hi); \
-
-
-__attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair
-kernel void kernel_gemm_moe_q4_0_f32_ns(
-        __read_only  image1d_buffer_t src0_q,
-        __global     half *           src0_d,
-        __read_only  image1d_buffer_t src1,
-        __global     uint *           src2,
-        __global     ushort *         src2_emap,
-        __write_only image1d_buffer_t dst,
-        __global     int *            total_tiles,
-        uint ne00,
-        uint ne01
-) {
-    uint block_id_m = get_global_id(1); // m_tile
-    uint block_id_n = get_global_id(2); // n_tile
-
-    // Boundary check
-    if (((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) || (block_id_n >= total_tiles[0])) {
-        return;
-    }
-
-    __private half16 reg_a;
-    __private float32 reg_c = (float32)(0);
-    __local half4 shared_b[128];
-
-    const ushort expert_id = src2_emap[block_id_n];
-
-    const uint row = block_id_m * TILESIZE_M;
-    const uint col = block_id_n * TILESIZE_N;
-
-    uint sub_block_id_m = get_local_id(0);
-    uint2 b_global_offset;
-    b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00;
-    b_global_offset.y = b_global_offset.x + (16 * ne00);
-    uint2 b_local_offset;
-    b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2);
-    b_local_offset.y = b_local_offset.x + 16;
-
-    // Loop along K axis, 32 elements (one block) for each iteration, divided into 2 sub-blocks
-    for (uint step = 0; step < ne00; step += TILESIZE_K * 2) {
-        // First sub-block
-        uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);
-        uint s_sub_offset = row + ((ne01 * step) >> 5) + ((expert_id * ne00 * ne01) >> 5);
-        uint b_sub_offset = col * ne00 + step;
-
-        // Load scale for current Q4_0 block
-        uint s_offset = s_sub_offset + get_global_id(0);
-        half s = src0_d[s_offset];
-
-        // Load 16 q (64-bits) in transposed layout
-        uint2 q4x16;
-        q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x;
-        q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x;
-
-        // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements
-        float8 bx8_f32;
-        bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4);
-        bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4);
-        // Convert to half and store to LM to share within the subgroup
-        half8 bx8_f16 = convert_half8(bx8_f32);
-        shared_b[b_local_offset.x] = bx8_f16.lo;
-        shared_b[b_local_offset.y] = bx8_f16.hi;
-
-        // Dequantization
-        dequantize_q4_0(as_ushort4(q4x16), reg_a, s);
-
-        sub_group_barrier(CLK_LOCAL_MEM_FENCE);
-
-        // 32 16x16 fp16 dot product with 8 elements reduction for better precision
-        half16 acc;
-        dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0);
-        dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16);
-
-        // Repeat for second sub-block
-        uint half_step = step + TILESIZE_K;
-        q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3);
-        b_sub_offset = col * ne00 + half_step;
-
-        // Load next 16 q (64-bits) in transposed layout
-        q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x;
-        q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x;
-
-        // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements
-        bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4);
-        bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4);
-        // Convert to half and store to LM to share within the subgroup
-        bx8_f16 = convert_half8(bx8_f32);
-        shared_b[b_local_offset.x] = bx8_f16.lo;
-        shared_b[b_local_offset.y] = bx8_f16.hi;
-
-        // Dequantization
-        dequantize_q4_0(as_ushort4(q4x16), reg_a, s);
-
-        sub_group_barrier(CLK_LOCAL_MEM_FENCE);
-
-        // 32 16x16 fp16 dot product with 3-levels reduction for better precision
-        dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0);
-        dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16);
-    }
-
-    // Load poster router and share in LM
-    __local uint out_idx[TILESIZE_N];
-
-    if (get_local_id(0) < TILESIZE_N) {
-        uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)];
-        if (idx == 0xFFFFFFFF) {
-            idx = src2[block_id_n * TILESIZE_N + 0];
-        }
-        out_idx[get_local_id(0)] = idx * ne01;
-    }
-
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    // Scatter results back to original position in output grid
-    uint m_offset = row + get_local_id(0);
-
-    write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1));
-    write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2));
-    write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3));
-    write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4));
-    write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5));
-    write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6));
-    write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7));
-    write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8));
-    write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9));
-    write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa));
-    write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb));
-    write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc));
-    write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd));
-    write_imagef(dst, out_idx[14] + m_offset, (reg_c.se));
-    write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf));
-    write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg));
-    write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh));
-    write_imagef(dst, out_idx[18] + m_offset, (reg_c.si));
-    write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj));
-    write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk));
-    write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl));
-    write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm));
-    write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn));
-    write_imagef(dst, out_idx[24] + m_offset, (reg_c.so));
-    write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp));
-    write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq));
-    write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr));
-    write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss));
-    write_imagef(dst, out_idx[29] + m_offset, (reg_c.st));
-    write_imagef(dst, out_idx[30] + m_offset, (reg_c.su));
-    write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv));
-
-    // Store zero padding parts to the index of first output in tile, override correct result in the end
-    barrier(CLK_GLOBAL_MEM_FENCE);
-    write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0));
-}

ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl

@@ -1,116 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-
-#define QK_Q4_0 32
-#define N_SIMDGROUP 4
-#define SIMDGROUP_WIDTH 64
-
-static inline float8 q4_0_to_fp32_packed8(ushort2 q4x8) {
-    float8 fp32x8;
-    fp32x8.s0 = (float)((q4x8.s0 & 0x000F) - 8);
-    fp32x8.s1 = (float)(((q4x8.s0 & 0x00F0) >> 4) - 8);
-    fp32x8.s2 = (float)(((q4x8.s0 & 0x0F00) >> 8) - 8);
-    fp32x8.s3 = (float)(((q4x8.s0 & 0xF000) >> 12) - 8);
-    fp32x8.s4 = (float)((q4x8.s1 & 0x000F) - 8);
-    fp32x8.s5 = (float)(((q4x8.s1 & 0x00F0) >> 4) - 8);
-    fp32x8.s6 = (float)(((q4x8.s1 & 0x0F00) >> 8) - 8);
-    fp32x8.s7 = (float)(((q4x8.s1 & 0xF000) >> 12) - 8);
-    return fp32x8;
-}
-
-
-__attribute__((qcom_reqd_sub_group_size("half")))
-__kernel void kernel_gemv_moe_q4_0_f32_ns(
-    __global uint * src0_q,
-    __global half * src0_d,
-    __read_only image1d_buffer_t src1,
-    __global uint * src2,
-    __global float * dst,
-    ulong         offsetd,
-    int           ne00,
-    int           ne01,
-    int           ne11
-) {
-    uint i01  = get_global_id(0);
-    uint i20  = get_global_id(2);
-    uint sgid = get_local_id(1);
-    uint slid = get_sub_group_local_id();
-
-    uint i11 = i20 % ne11;
-
-    uint expert_id = src2[i20];
-    uint expert_offset = expert_id * ne00 * ne01 / 32;
-
-    __private float sum = 0.0f; // each thread calculate partial sum of one output
-
-    // loop along ne00 in block granularity, skip 4 blocks every iter
-    for (uint ib00 = sgid; ib00 < (ne00 / QK_Q4_0); ib00 += N_SIMDGROUP) {
-
-        // load one block of q
-        uint4 regQ;
-        uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01;
-
-        regQ.s0 = src0_q[block_offset];
-        regQ.s1 = src0_q[block_offset + ne01];
-        regQ.s2 = src0_q[block_offset + ne01 * 2];
-        regQ.s3 = src0_q[block_offset + ne01 * 3];
-
-        uint offset = i11 * ne00 / 4 + ib00 * 8;
-
-        float8 fp32x8 = q4_0_to_fp32_packed8(as_ushort2(regQ.s0));
-
-        float4 shared_y4;
-        shared_y4 = read_imagef(src1, (offset + 0));
-        float4 acc = shared_y4 * fp32x8.lo;
-
-        shared_y4 = read_imagef(src1, (offset + 1));
-        acc += shared_y4 * fp32x8.hi;
-
-        fp32x8 = q4_0_to_fp32_packed8(as_ushort2(regQ.s1));
-
-        shared_y4 = read_imagef(src1, (offset + 2));
-        acc += shared_y4 * fp32x8.lo;
-
-        shared_y4 = read_imagef(src1, (offset + 3));
-        acc += shared_y4 * fp32x8.hi;
-
-
-        fp32x8 = q4_0_to_fp32_packed8(as_ushort2(regQ.s2));
-
-        shared_y4 = read_imagef(src1, (offset + 4));
-        acc += shared_y4 * fp32x8.lo;
-
-        shared_y4 = read_imagef(src1, (offset + 5));
-        acc += shared_y4 * fp32x8.hi;
-
-
-        fp32x8 = q4_0_to_fp32_packed8(as_ushort2(regQ.s3));
-
-        shared_y4 = read_imagef(src1, (offset + 6));
-        acc += shared_y4 * fp32x8.lo;
-
-        shared_y4 = read_imagef(src1, (offset + 7));
-        acc += shared_y4 * fp32x8.hi;
-
-        half regS = src0_d[ib00 * ne01 + i01 + expert_offset];
-        sum += (float)(regS) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3));
-    }
-
-    // reduction in local memory, assumes #subgroups=4
-    __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)];
-    if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum;
-    if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum;
-    if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum;
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];
-    if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];
-    if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];
-
-    // 1 outputs per thread in subgroup 0
-    if (sgid == 0) {
-        dst = dst + (offsetd >> 2);
-        dst[i01 + i20 * ne01] = sum;
-    }
-
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl

@@ -191,7 +191,7 @@
 #ifdef ADRENO_GPU
 REQD_SUBGROUP_SIZE_64
 #endif
-__kernel void kernel_gemv_noshuffle_q4_0_f32(
+__kernel void kernel_gemv_noshuffle(
         __read_only  image1d_buffer_t src0_q,  // quantized A
         global half2  * src0_d,  // A scales
         __read_only  image1d_buffer_t src1,    // B
@@ -232,21 +232,21 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
         regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
         regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
         regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
-#ifdef VECTOR_SUB_GROUP_BROADCAST
+#ifdef VECTOR_SUB_GROUP_BROADCAT
         dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB);
 #else
         dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB);
-#endif // VECTOR_SUB_GROUP_BROADCAST
+#endif // VECTOR_SUB_GROUP_BROADCAT
 
         regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
         regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
         regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
         regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
-#ifdef VECTOR_SUB_GROUP_BROADCAST
+#ifdef VECTOR_SUB_GROUP_BROADCAT
         dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB);
 #else
         dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regB);
-#endif // VECTOR_SUB_GROUP_BROADCAST
+#endif // VECTOR_SUB_GROUP_BROADCAT
     }
 
     // reduction in local memory, assumes #wave=4

ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl

@@ -191,7 +191,7 @@
 #ifdef ADRENO_GPU
 REQD_SUBGROUP_SIZE_64
 #endif
-__kernel void kernel_gemv_noshuffle_q4_0_f32(
+__kernel void kernel_gemv_noshuffle(
         __read_only  image1d_buffer_t src0_q,  // quantized A
         global half2  * src0_d,  // A scales
         __read_only  image1d_buffer_t src1,    // B
@@ -238,21 +238,21 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
         regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
         regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
         regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
-#ifdef VECTOR_SUB_GROUP_BROADCAST
+#ifdef VECTOR_SUB_GROUP_BROADCAT
         dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB);
 #else
         dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB);
-#endif // VECTOR_SUB_GROUP_BROADCAST
+#endif // VECTOR_SUB_GROUP_BROADCAT
 
         regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
         regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
         regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
         regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
-#ifdef VECTOR_SUB_GROUP_BROADCAST
+#ifdef VECTOR_SUB_GROUP_BROADCAT
         dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB);
 #else
         dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regB);
-#endif // VECTOR_SUB_GROUP_BROADCAST
+#endif // VECTOR_SUB_GROUP_BROADCAT
     }
 
     // reduction in local memory, assumes #wave=4

ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl

@@ -17,7 +17,7 @@
 REQD_SUBGROUP_SIZE_128
 #endif
 
-kernel void kernel_gemm_noshuffle_q4_0_f32(
+kernel void kernel_mul_mat_Ab_Bi_8x4(
         global const ushort * src0_q,       // quantized A
         global const half  * src0_d,        // A scales
         __read_only image1d_buffer_t src1,  // B (1d image)

ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_8x4.cl

@@ -11,7 +11,7 @@
 REQD_SUBGROUP_SIZE_128
 #endif
 
-kernel void kernel_gemm_noshuffle_q8_0_f32(
+kernel void kernel_mul_mm_q8_0_f32_8x4(
         global const uint * src0_q,
         global const half  * src0_d,
         __read_only image1d_buffer_t src1,

ggml/src/ggml-rpc/ggml-rpc.cpp

@@ -207,11 +207,35 @@ struct ggml_backend_rpc_buffer_type_context {
     size_t      max_size;
 };
 
+struct graph_cache {
+
+    bool is_cached(const ggml_cgraph * cgraph) {
+        if ((int)last_graph.size() != cgraph->n_nodes) {
+            return false;
+        }
+        for (int i = 0; i < cgraph->n_nodes; i++) {
+            if (memcmp(&last_graph[i], cgraph->nodes[i], sizeof(ggml_tensor)) != 0) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    void add(const ggml_cgraph * cgraph) {
+        last_graph.resize(cgraph->n_nodes);
+        for (int i = 0; i < cgraph->n_nodes; i++) {
+            memcpy(&last_graph[i], cgraph->nodes[i], sizeof(ggml_tensor));
+        }
+    }
+
+    std::vector<ggml_tensor> last_graph;
+};
+
 struct ggml_backend_rpc_context {
     std::string endpoint;
     uint32_t    device;
     std::string name;
-    uint64_t    last_graph_uid;
+    graph_cache gc;
 };
 
 struct ggml_backend_rpc_buffer_context {
@@ -693,7 +717,7 @@ static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, g
     ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;
 
     GGML_ASSERT(cgraph->n_nodes > 0);
-    bool reuse = cgraph->uid != 0 && rpc_ctx->last_graph_uid == cgraph->uid;
+    bool reuse = rpc_ctx->gc.is_cached(cgraph);
     if (reuse) {
         rpc_msg_graph_recompute_req request;
         request.device = rpc_ctx->device;
@@ -701,7 +725,7 @@ static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, g
         bool status = send_rpc_cmd(sock, RPC_CMD_GRAPH_RECOMPUTE, &request, sizeof(request));
         RPC_STATUS_ASSERT(status);
     } else {
-        rpc_ctx->last_graph_uid = cgraph->uid;
+        rpc_ctx->gc.add(cgraph);
         std::vector<uint8_t> input;
         serialize_graph(rpc_ctx->device, cgraph, input);
         auto sock = get_socket(rpc_ctx->endpoint);
@@ -767,10 +791,10 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u
 ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device) {
     std::string dev_name = "RPC" + std::to_string(device) + "[" + std::string(endpoint) + "]";
     ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context {
-        /* .endpoint       = */ endpoint,
-        /* .device         = */ device,
-        /* .name           = */ dev_name,
-        /* .last_graph_uid = */ 0,
+        /* .endpoint = */ endpoint,
+        /* .device   = */ device,
+        /* .name     = */ dev_name,
+        /* .gc       = */ {},
     };
     auto reg = ggml_backend_rpc_add_server(endpoint);
     ggml_backend_t backend = new ggml_backend {

ggml/src/ggml-sycl/CMakeLists.txt

@@ -135,11 +135,7 @@ endif()
 
 if (GGML_SYCL_TARGET STREQUAL "INTEL")
     add_compile_definitions(GGML_SYCL_WARP_SIZE=16)
-    if (NOT GGML_SYCL_DEVICE_ARCH)
-        target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required)
-    else()
-        message(STATUS "Skipping -ze-intel-greater-than-4GB-buffer-required for spir64_gen AOT")
-    endif()
+    target_link_options(ggml-sycl PRIVATE  -Xs   -ze-intel-greater-than-4GB-buffer-required)
 
     # Link against Intel oneMKL
     if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
@@ -164,15 +160,7 @@ if (GGML_SYCL_HOST_MEM_FALLBACK)
 endif()
 
 if (GGML_SYCL_DEVICE_ARCH)
-    message(STATUS "GGML_SYCL_DEVICE_ARCH=${GGML_SYCL_DEVICE_ARCH} (AOT via spir64_gen)")
-    target_compile_options(
-        ggml-sycl PRIVATE
-        -fsycl-targets=spir64_gen
-        "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\""
-    )
-    target_link_options(
-        ggml-sycl PRIVATE
-        -fsycl-targets=spir64_gen
-        "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\""
-    )
+    target_compile_options(ggml-sycl PRIVATE -Xsycl-target-backend --offload-arch=${GGML_SYCL_DEVICE_ARCH})
+    target_link_options(ggml-sycl PRIVATE -Xsycl-target-backend --offload-arch=${GGML_SYCL_DEVICE_ARCH})
 endif()
+

ggml/src/ggml-sycl/common.hpp

@@ -25,7 +25,6 @@
 #include "presets.hpp"
 #include "type.hpp"
 #include "sycl_hw.hpp"
-#include "fattn-buffers.hpp"
 
 namespace syclexp = sycl::ext::oneapi::experimental;
 
@@ -405,16 +404,12 @@ struct ggml_backend_sycl_context {
     std::unique_ptr<ggml_sycl_pool> pools[GGML_SYCL_MAX_DEVICES];
     std::unordered_map<sycl::queue *, std::unique_ptr<ggml_sycl_pool_alloc<uint8_t>>> scratchpad_map;
 
-    std::unique_ptr<ggml_sycl_fattn_kv_buffers> fattn_bufs[GGML_SYCL_MAX_DEVICES];
-
     std::unique_ptr<ggml_sycl_pool> host_pools[GGML_SYCL_MAX_DEVICES];
 
     static std::unique_ptr<ggml_sycl_pool> new_pool_for_device(queue_ptr qptr, int device);
 
     static std::unique_ptr<ggml_sycl_pool> new_pool_for_host(queue_ptr qptr, int device);
 
-    static std::unique_ptr<ggml_sycl_fattn_kv_buffers> new_fattn_kv_buffers(queue_ptr qptr, int device);
-
     ggml_sycl_pool & pool(int device) {
         if (pools[device] == nullptr) {
             pools[device] = new_pool_for_device(stream(device,0), device);
@@ -426,17 +421,6 @@ struct ggml_backend_sycl_context {
         return pool(device);
     }
 
-    ggml_sycl_fattn_kv_buffers & fattn_buffers(int device) {
-        if (fattn_bufs[device] == nullptr) {
-            fattn_bufs[device] = new_fattn_kv_buffers(stream(device, 0), device);
-        }
-        return *fattn_bufs[device];
-    }
-
-    ggml_sycl_fattn_kv_buffers & fattn_buffers() {
-        return fattn_buffers(device);
-    }
-
 #ifdef GGML_SYCL_GRAPH
     std::unique_ptr<sycl_ex::command_graph<sycl_ex::graph_state::executable>> exec_graph = nullptr;
 #endif

ggml/src/ggml-sycl/convert.cpp

@@ -252,23 +252,6 @@ static void dequantize_row_q5_K_sycl(const void *vx, dst_t *y, const int64_t k,
 #endif
 }
 
-template <typename dst_t>
-static void dequantize_row_q5_K_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {
-    const int64_t nb = k / QK_K;
-
-    dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
-
-    stream->submit([&](sycl::handler & cgh) {
-        sycl::local_accessor<uint8_t, 1> scale_local_acc(sycl::range<1>(K_SCALE_SIZE), cgh);
-
-        cgh.parallel_for(
-            sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)),
-            [=](sycl::nd_item<3> item_ct1) {
-                dequantize_block_q5_K_reorder(vx, y, get_pointer(scale_local_acc), item_ct1, nb);
-            });
-    });
-}
-
 template <typename dst_t>
 static void dequantize_row_q6_K_sycl(const void *vx, dst_t *y, const int64_t k,
                                      dpct::queue_ptr stream) {
@@ -660,11 +643,7 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
                 return dequantize_row_q4_K_sycl;
             }
         case GGML_TYPE_Q5_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q5_K_sycl_reorder;
-            } else {
-                return dequantize_row_q5_K_sycl;
-            }
+            return dequantize_row_q5_K_sycl;
         case GGML_TYPE_Q6_K:
             if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
                 return dequantize_row_q6_K_sycl_reorder;
@@ -739,11 +718,7 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) {
                 return dequantize_row_q4_K_sycl;
             }
         case GGML_TYPE_Q5_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q5_K_sycl_reorder;
-            } else {
-                return dequantize_row_q5_K_sycl;
-            }
+            return dequantize_row_q5_K_sycl;
         case GGML_TYPE_Q6_K:
             if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
                 return dequantize_row_q6_K_sycl_reorder;

ggml/src/ggml-sycl/cumsum.cpp

@@ -1,148 +0,0 @@
-#include "cumsum.hpp"
-#include "common.hpp"
-
-#include <algorithm>
-
-#define SYCL_CUMSUM_BLOCK_SIZE 256
-
-static __dpct_inline__ float warp_prefix_inclusive_sum_f32(float x, const sycl::nd_item<3> & item) {
-    return sycl::inclusive_scan_over_group(item.get_sub_group(), x, sycl::plus<float>());
-}
-
-static void cumsum_f32_kernel(
-        const float * __restrict__ src, float * __restrict__ dst,
-        const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
-        const int64_t s01, const int64_t s02, const int64_t s03,
-        const int64_t  d1, const int64_t  d2, const int64_t  d3,
-        const sycl::nd_item<3> & item, float * smem) {
-
-    const int tid = item.get_local_id(2);
-    const int block_size = item.get_local_range(2);
-    const int lane = tid % WARP_SIZE;
-    const int warp = tid / WARP_SIZE;
-    const int warps_per_block = block_size / WARP_SIZE;
-
-    float * s_vals      = smem;
-    float * s_warp_sums = smem + block_size;
-    float * s_carry     = smem + block_size + warps_per_block;
-
-    if (tid == 0) {
-        s_carry[0] = 0.0f;
-    }
-    item.barrier(sycl::access::fence_space::local_space);
-
-    const int64_t i3 = item.get_group(0);
-    const int64_t i2 = item.get_group(1);
-    const int64_t i1 = item.get_group(2);
-    if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {
-        return;
-    }
-
-    const float * src_row = src + i1 * s01 + i2 * s02 + i3 * s03;
-    float       * dst_row = dst + i1 * d1  + i2 * d2  + i3 * d3;
-
-    constexpr int num_unroll = 4;
-    float temp[num_unroll];
-
-    for (int64_t i = 0; i < ne00; i += num_unroll * block_size) {
-        int64_t idx = i + tid * num_unroll;
-
-        temp[0] = (idx < ne00 ? src_row[idx] : 0.0f);
-#pragma unroll
-        for (int j = 1; j < num_unroll; j++) {
-            temp[j] = temp[j - 1];
-            if (idx + j < ne00) {
-                temp[j] += src_row[idx + j];
-            }
-        }
-
-        float val = (idx < ne00) ? temp[num_unroll - 1] : 0.0f;
-
-        val = warp_prefix_inclusive_sum_f32(val, item);
-        s_vals[tid] = val;
-
-        if (lane == WARP_SIZE - 1) {
-            s_warp_sums[warp] = val;
-        }
-        item.barrier(sycl::access::fence_space::local_space);
-
-        if (warp == 0) {
-            float w = (tid < warps_per_block) ? s_warp_sums[tid] : 0.0f;
-            float inc = warp_prefix_inclusive_sum_f32(w, item);
-            if (tid < warps_per_block) {
-                s_warp_sums[tid] = inc - w;
-            }
-            if (tid == warps_per_block - 1) {
-                s_carry[1] = inc;
-            }
-        }
-        item.barrier(sycl::access::fence_space::local_space);
-
-        float carry = s_carry[0];
-        float final_offset = s_vals[tid] + s_warp_sums[warp] + carry - temp[num_unroll - 1];
-
-#pragma unroll
-        for (int j = 0; j < num_unroll; j++) {
-            if (idx + j < ne00) {
-                dst_row[idx + j] = temp[j] + final_offset;
-            }
-        }
-
-        item.barrier(sycl::access::fence_space::local_space);
-
-        if (tid == 0) {
-            s_carry[0] += s_carry[1];
-        }
-    }
-}
-
-inline void ggml_sycl_op_cumsum(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    const ggml_tensor * src0 = dst->src[0];
-
-    GGML_ASSERT(src0->type == GGML_TYPE_F32);
-    GGML_ASSERT(dst->type == GGML_TYPE_F32);
-
-    dpct::queue_ptr stream = ctx.stream();
-    SYCL_CHECK(ggml_sycl_set_device(ctx.device));
-
-    const float * src_d = static_cast<const float *>(src0->data);
-    float       * dst_d = static_cast<float *>(dst->data);
-
-    const int64_t ne00 = src0->ne[0];
-    const int64_t ne01 = src0->ne[1];
-    const int64_t ne02 = src0->ne[2];
-    const int64_t ne03 = src0->ne[3];
-
-    const size_t ts = sizeof(float);
-    const int64_t s01 = src0->nb[1] / ts;
-    const int64_t s02 = src0->nb[2] / ts;
-    const int64_t s03 = src0->nb[3] / ts;
-    const int64_t d1  = dst->nb[1] / ts;
-    const int64_t d2  = dst->nb[2] / ts;
-    const int64_t d3  = dst->nb[3] / ts;
-
-    const int num_warps = (ne00 + WARP_SIZE - 1) / WARP_SIZE;
-    int block_size = num_warps * WARP_SIZE;
-    block_size = std::min(block_size, SYCL_CUMSUM_BLOCK_SIZE);
-    const int warps_per_block = block_size / WARP_SIZE;
-    const int smem_size = block_size + warps_per_block + 2;
-
-    const sycl::range<3> grid(ne03, ne02, ne01);
-    const sycl::range<3> block(1, 1, block_size);
-
-    stream->submit([&](sycl::handler & cgh) {
-        sycl::local_accessor<float, 1> smem_acc(sycl::range<1>(smem_size), cgh);
-        cgh.parallel_for(
-            sycl::nd_range<3>(grid * block, block),
-            [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                cumsum_f32_kernel(src_d, dst_d, ne00, ne01, ne02, ne03,
-                                  s01, s02, s03, d1, d2, d3,
-                                  item, get_pointer(smem_acc));
-            });
-    });
-}
-
-void ggml_sycl_cumsum(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
-    ggml_sycl_op_cumsum(ctx, dst);
-}

ggml/src/ggml-sycl/cumsum.hpp

@@ -1,5 +0,0 @@
-#pragma once
-
-#include "common.hpp"
-
-void ggml_sycl_cumsum(ggml_backend_sycl_context & ctx, ggml_tensor * dst);

ggml/src/ggml-sycl/dequantize.hpp

@@ -537,63 +537,6 @@ static void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restri
 #endif
 }
 
-template <typename dst_t>
-static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,
-                                          uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {
-    const int64_t ib = item_ct1.get_group(2);
-
-#if QK_K == 256
-    // assume 64 threads
-    const int64_t tid = item_ct1.get_local_id(2);
-    const int64_t il  = tid / 16;   // 0...3
-    const int64_t ir  = tid % 16;   // 0...15
-    const int64_t is  = 2 * il;
-
-    dst_t * y = yy + ib * QK_K + 64 * il + 2 * ir;
-
-    const uint8_t * base = static_cast<const uint8_t *>(vx);
-
-    // Reordered layout: [qs (QK_K/2 per block)] [qh (QK_K/8 per block)] [scales (K_SCALE_SIZE per block)] [dm (half2 per block)]
-    const size_t qs_offset     = ib * (QK_K / 2);
-    const size_t qh_offset     = n_blocks * (QK_K / 2) + ib * (QK_K / 8);
-    const size_t scales_offset = n_blocks * (QK_K / 2) + n_blocks * (QK_K / 8) + ib * K_SCALE_SIZE;
-    const size_t dm_offset     = n_blocks * (QK_K / 2) + n_blocks * (QK_K / 8) + n_blocks * K_SCALE_SIZE + ib * sizeof(ggml_half2);
-
-    const uint8_t *  qs_ptr     = base + qs_offset;
-    const uint8_t *  qh_ptr     = base + qh_offset;
-    const uint8_t *  scales_ptr = base + scales_offset;
-    const ggml_half2 dm_values  = *reinterpret_cast<const ggml_half2 *>(base + dm_offset);
-
-    const float dall = dm_values.x();
-    const float dmin = dm_values.y();
-
-    const uint8_t * ql = qs_ptr + 32 * il + 2 * ir;
-    const uint8_t * qh = qh_ptr + 2 * ir;
-
-    if (tid < K_SCALE_SIZE) {
-        scales_local[tid] = scales_ptr[tid];
-    }
-
-    item_ct1.barrier(sycl::access::fence_space::local_space);
-
-    uint8_t sc, m;
-    get_scale_min_k4(is + 0, scales_local, sc, m);
-    const float d1 = dall * sc; const float m1 = dmin * m;
-    get_scale_min_k4(is + 1, scales_local, sc, m);
-    const float d2 = dall * sc; const float m2 = dmin * m;
-
-    uint8_t hm  = 1 << (2 * il);
-    y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;
-    y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;
-    hm <<= 1;
-    y[32] = d2 * ((ql[ 0] >>  4) + (qh[ 0] & hm ? 16 : 0)) - m2;
-    y[33] = d2 * ((ql[ 1] >>  4) + (qh[ 1] & hm ? 16 : 0)) - m2;
-#else
-    GGML_UNUSED(ib); GGML_UNUSED(tid); GGML_UNUSED(yy); GGML_UNUSED(scales_local); GGML_UNUSED(n_blocks);
-    GGML_ABORT("Q5_K reorder dequantize not supported for QK_K != 256");
-#endif
-}
-
 template<typename dst_t>
 static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy,
                                   const sycl::nd_item<3> &item_ct1) {

ggml/src/ggml-sycl/diag.cpp

@@ -1,67 +0,0 @@
-#include "diag.hpp"
-#include "common.hpp"
-
-#define SYCL_DIAG_BLOCK_SIZE 256
-
-template <typename T>
-static void diag_kernel(T * __restrict__ dst, const T * __restrict__ src,
-                        const int64_t ne0, const int64_t ne1,
-                        const int64_t ne2, const int64_t ne3,
-                        const int64_t total_elements,
-                        const sycl::nd_item<1> & item) {
-    const int64_t i = item.get_global_id(0);
-    if (i >= total_elements) {
-        return;
-    }
-
-    const int64_t i0 = i % ne0;
-    const int64_t i1 = (i / ne0) % ne1;
-    const int64_t i2 = (i / (ne0 * ne1)) % ne2;
-    const int64_t i3 = i / (ne0 * ne1 * ne2);
-
-    const int64_t dst_idx = ((i3 * ne2 + i2) * ne1 + i1) * ne0 + i0;
-
-    if (i0 == i1) {
-        const int64_t batch_idx = i3 * ne2 + i2;
-        dst[dst_idx] = src[batch_idx * ne0 + i0];
-    } else {
-        dst[dst_idx] = T(0);
-    }
-
-    (void)ne3;
-}
-
-inline void ggml_sycl_op_diag(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    const ggml_tensor * src0 = dst->src[0];
-
-    GGML_ASSERT(ggml_is_contiguous(dst));
-    GGML_ASSERT(ggml_is_contiguous(src0));
-    GGML_ASSERT(src0->ne[1] == 1);
-
-    dpct::queue_ptr stream = ctx.stream();
-    SYCL_CHECK(ggml_sycl_set_device(ctx.device));
-
-    const void * src0_d = src0->data;
-    void * dst_d = dst->data;
-
-    const int64_t ne0 = dst->ne[0];
-    const int64_t ne1 = dst->ne[1];
-    const int64_t ne2 = dst->ne[2];
-    const int64_t ne3 = dst->ne[3];
-    const int64_t n_elems = ggml_nelements(dst);
-    const int64_t num_blocks = (n_elems + SYCL_DIAG_BLOCK_SIZE - 1) / SYCL_DIAG_BLOCK_SIZE;
-
-    GGML_ASSERT(dst->type == GGML_TYPE_F32);
-    stream->parallel_for(
-        sycl::nd_range<1>(num_blocks * SYCL_DIAG_BLOCK_SIZE, SYCL_DIAG_BLOCK_SIZE),
-        [=](sycl::nd_item<1> item) {
-            diag_kernel(static_cast<float *>(dst_d),
-                        static_cast<const float *>(src0_d),
-                        ne0, ne1, ne2, ne3, n_elems, item);
-        });
-}
-
-void ggml_sycl_diag(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
-    ggml_sycl_op_diag(ctx, dst);
-}

ggml/src/ggml-sycl/diag.hpp

@@ -1,5 +0,0 @@
-#pragma once
-
-#include "common.hpp"
-
-void ggml_sycl_diag(ggml_backend_sycl_context & ctx, ggml_tensor * dst);

ggml/src/ggml-sycl/fattn-buffers.cpp

@@ -1,56 +0,0 @@
-//
-// MIT license
-// Copyright (C) 2025 Intel Corporation
-// SPDX-License-Identifier: MIT
-//
-
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-
-#include "common.hpp"
-
-sycl::half * ggml_sycl_fattn_kv_buffers::kv_buffer::ensure_half(size_t n_elems) {
-    const size_t need_bytes = n_elems * sizeof(sycl::half);
-
-    if (capacity >= need_bytes) {
-        return ptr;
-    }
-
-    if (ptr) {
-        SYCL_CHECK(CHECK_TRY_ERROR(qptr->wait()));
-        SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr)));
-        ptr = nullptr;
-        capacity = 0;
-    }
-
-    size_t cap = 0;
-    while (cap < need_bytes) {
-        cap += CHUNK_SIZE;
-    }
-
-    void * dev_ptr;
-    SYCL_CHECK(
-        CHECK_TRY_ERROR(dev_ptr = sycl::malloc_device(
-                        cap, *qptr)));
-
-    if (!dev_ptr) {
-        GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, cap);
-        GGML_ABORT("fattn buffer alloc failed");
-    }
-
-    ptr = static_cast<sycl::half *>(dev_ptr);
-    capacity = cap;
-    return ptr;
-}
-
-ggml_sycl_fattn_kv_buffers::kv_buffer::~kv_buffer() {
-#ifdef DEBUG_SYCL_POOL
-    GGML_LOG_INFO("ggml_sycl_fattn_kv_buffer[%d]: %.2f MiB\n", device, capacity / 1024.0 / 1024.0);
-#endif
-    if (ptr) {
-        SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr)));
-    }
-}

ggml/src/ggml-sycl/fattn-buffers.hpp

@@ -1,63 +0,0 @@
-//
-// MIT license
-// Copyright (C) 2025 Intel Corporation
-// SPDX-License-Identifier: MIT
-//
-
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-
-#ifndef GGML_SYCL_FATTN_BUFFERS_HPP
-#define GGML_SYCL_FATTN_BUFFERS_HPP
-
-#include <sycl/sycl.hpp>
-
-typedef sycl::queue *queue_ptr;
-
-struct ggml_sycl_fattn_kv_buffers {
-    // buffers grow in chunks of this size
-    static constexpr size_t CHUNK_SIZE = 16ull << 20; // 16 MiB
-
-    struct kv_buffer {
-        kv_buffer(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {}
-        ~kv_buffer();
-
-        kv_buffer(const kv_buffer &) = delete;
-        kv_buffer & operator=(const kv_buffer &) = delete;
-
-        sycl::half * ensure_half(size_t n_elems);
-
-    private:
-        sycl::half * ptr      = nullptr;
-        size_t       capacity = 0;
-        queue_ptr    qptr     = nullptr;
-        [[maybe_unused]] int device = 0;
-    };
-
-    kv_buffer K;
-    kv_buffer V;
-
-    ggml_sycl_fattn_kv_buffers(queue_ptr qptr, int device) : K(qptr, device), V(qptr, device) {}
-
-    ggml_sycl_fattn_kv_buffers(const ggml_sycl_fattn_kv_buffers &) = delete;
-    ggml_sycl_fattn_kv_buffers & operator=(const ggml_sycl_fattn_kv_buffers &) = delete;
-};
-
-/**
- * Imitates `ggml_sycl_pool_alloc` to keep the code calling alloc unchanged.
- */
-struct ggml_sycl_fattn_alloc {
-    ggml_sycl_fattn_kv_buffers::kv_buffer & buf;
-    sycl::half *                         ptr = nullptr;
-
-    explicit ggml_sycl_fattn_alloc(ggml_sycl_fattn_kv_buffers::kv_buffer & buf_) : buf(buf_) {}
-
-    sycl::half * alloc(size_t n_elems) {
-        ptr = buf.ensure_half(n_elems);
-        return ptr;
-    }
-};
-#endif

ggml/src/ggml-sycl/fattn-common.hpp

@@ -5,7 +5,6 @@
 #include "common.hpp"
 #include "convert.hpp"
 #include "vecdotq.hpp"
-#include "fattn-buffers.hpp"
 
 #include "ggml.h"
 
@@ -919,13 +918,12 @@ void launch_fattn(
     GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16);
 
     ggml_sycl_pool & pool = ctx.pool();
-    ggml_sycl_fattn_kv_buffers & fbuf = ctx.fattn_buffers();
     dpct::queue_ptr  main_stream = ctx.stream();
     const int id  = ggml_sycl_get_device();
     const int nsm = ggml_sycl_info().devices[id].nsm;
 
-    ggml_sycl_fattn_alloc        K_f16(fbuf.K);
-    ggml_sycl_fattn_alloc        V_f16(fbuf.V);
+    ggml_sycl_pool_alloc<sycl::half>   K_f16(pool);
+    ggml_sycl_pool_alloc<sycl::half>   V_f16(pool);
     ggml_sycl_pool_alloc<int>    KV_max(pool);
     ggml_sycl_pool_alloc<float>  dst_tmp(pool);
     ggml_sycl_pool_alloc<sycl::float2> dst_tmp_meta(pool);

ggml/src/ggml-sycl/fill.cpp

@@ -1,55 +0,0 @@
-#include "fill.hpp"
-#include "common.hpp"
-
-#define SYCL_FILL_BLOCK_SIZE 256
-
-template <typename T>
-static void fill_kernel(T * dst, const int64_t k, const T value,
-                        const sycl::nd_item<1> & item) {
-    const int64_t i = (int64_t)item.get_global_id(0);
-    if (i >= k) {
-        return;
-    }
-    dst[i] = value;
-}
-
-inline void ggml_sycl_op_fill(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    GGML_ASSERT(ggml_is_contiguous(dst));
-
-    dpct::queue_ptr stream = ctx.stream();
-    SYCL_CHECK(ggml_sycl_set_device(ctx.device));
-
-    float value;
-    memcpy(&value, dst->op_params, sizeof(float));
-
-    const int64_t k = ggml_nelements(dst);
-    const int64_t num_blocks = (k + SYCL_FILL_BLOCK_SIZE - 1) / SYCL_FILL_BLOCK_SIZE;
-    void * dst_d = dst->data;
-
-    switch (dst->type) {
-        case GGML_TYPE_F32:
-            stream->parallel_for(
-                sycl::nd_range<1>(num_blocks * SYCL_FILL_BLOCK_SIZE, SYCL_FILL_BLOCK_SIZE),
-                [=](sycl::nd_item<1> item) {
-                    fill_kernel(static_cast<float *>(dst_d), k, value, item);
-                });
-            break;
-        case GGML_TYPE_F16:
-            {
-                sycl::half h_value = sycl::half(value);
-                stream->parallel_for(
-                    sycl::nd_range<1>(num_blocks * SYCL_FILL_BLOCK_SIZE, SYCL_FILL_BLOCK_SIZE),
-                    [=](sycl::nd_item<1> item) {
-                        fill_kernel(static_cast<sycl::half *>(dst_d), k, h_value, item);
-                    });
-            }
-            break;
-        default:
-            GGML_ABORT("unsupported type");
-    }
-}
-
-void ggml_sycl_fill(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/0);
-    ggml_sycl_op_fill(ctx, dst);
-}

ggml/src/ggml-sycl/fill.hpp

@@ -1,5 +0,0 @@
-#pragma once
-
-#include "common.hpp"
-
-void ggml_sycl_fill(ggml_backend_sycl_context & ctx, ggml_tensor * dst);

ggml/src/ggml-sycl/gated_delta_net.hpp

@@ -5,5 +5,4 @@
 #include "common.hpp"
 #include "ggml.h"
 
-void ggml_sycl_op_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
 void ggml_sycl_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst);

ggml/src/ggml-sycl/getrows.cpp

@@ -183,10 +183,6 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
             get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const sycl::half *)dst->src[0]->data,
                                 src1_i32, (float *)dst->data, ctx.stream());
             break;
-        case GGML_TYPE_BF16:
-            get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const sycl::ext::oneapi::bfloat16 *)dst->src[0]->data,
-                                src1_i32, (float *)dst->data, ctx.stream());
-            break;
         case GGML_TYPE_F32:
             get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
             src1_i32, (float *)dst->data, ctx.stream());

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -54,12 +54,7 @@
 #include "ggml-sycl/set.hpp"
 #include "ggml-sycl/ssm_conv.hpp"
 #include "ggml-sycl/sycl_hw.hpp"
-#include "ggml-sycl/ssm_scan.hpp"
-#include "ggml-sycl/fill.hpp"
-#include "ggml-sycl/cumsum.hpp"
-#include "ggml-sycl/diag.hpp"
-#include "ggml-sycl/solve_tri.hpp"
-#include "ggml-sycl/gated_delta_net.hpp"
+
 
 static bool g_sycl_loaded = false;
 int g_ggml_sycl_debug = 0;
@@ -1286,23 +1281,6 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool {
     explicit ggml_sycl_pool_leg(queue_ptr qptr_, int device_) : device(device_), qptr(qptr_) {}
 
     ~ggml_sycl_pool_leg() {
-#ifdef DEBUG_SYCL_POOL
-        int    n_cached    = 0;
-        size_t bytes_cached = 0;
-        for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) {
-            if (buffer_pool[i].ptr != nullptr) {
-                ++n_cached;
-                bytes_cached += buffer_pool[i].size;
-            }
-        }
-        GGML_LOG_INFO("%s: %d buffers, cached = %.2f MiB\n", __func__,
-                      n_cached, bytes_cached / 1024.0 / 1024.0);
-        const auto slots = format_slots_in_alloc_order();
-        if (!slots.empty()) {
-            GGML_LOG_INFO("%s: slots MiB: %s\n", __func__, slots.c_str());
-        }
-#endif
-
         for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) {
             ggml_sycl_buffer & b = buffer_pool[i];
             if (b.ptr != nullptr) {
@@ -1313,26 +1291,6 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool {
         GGML_ASSERT(pool_size == 0);
     }
 
-#ifdef DEBUG_SYCL_POOL
-    std::string format_slots_in_alloc_order() const {
-        std::string line;
-        char buf[32];
-        bool first = true;
-        for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) {
-            if (buffer_pool[i].ptr == nullptr) {
-                continue;
-            }
-            if (!first) {
-                line += '/';
-            }
-            first = false;
-            snprintf(buf, sizeof(buf), "%.2f", buffer_pool[i].size / 1024.0 / 1024.0);
-            line += buf;
-        }
-        return line;
-    }
-#endif
-
     void * alloc(size_t size, size_t * actual_size) override {
 #ifdef DEBUG_sycl_MALLOC
         int nnz = 0;
@@ -1496,10 +1454,6 @@ std::unique_ptr<ggml_sycl_pool> ggml_backend_sycl_context::new_pool_for_device(q
    return std::unique_ptr<ggml_sycl_pool>(new ggml_sycl_pool_leg(qptr, device));
 }
 
-std::unique_ptr<ggml_sycl_fattn_kv_buffers> ggml_backend_sycl_context::new_fattn_kv_buffers(queue_ptr qptr, int device) {
-    return std::unique_ptr<ggml_sycl_fattn_kv_buffers>(new ggml_sycl_fattn_kv_buffers(qptr, device));
-}
-
 // TBD pool with virtual memory management
 // struct ggml_sycl_pool_vmm : public ggml_sycl_pool
 
@@ -3344,7 +3298,6 @@ inline bool ggml_sycl_supports_reorder_mul_mat_sycl(enum ggml_type type) {
         case GGML_TYPE_Q8_0:
             return true;
         case GGML_TYPE_Q4_K:
-        case GGML_TYPE_Q5_K:
         case GGML_TYPE_Q6_K:
             return !g_ggml_sycl_prioritize_dmmv;
         default:
@@ -3367,7 +3320,6 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) {
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_Q8_0:
         case GGML_TYPE_Q4_K:
-        case GGML_TYPE_Q5_K:
         case GGML_TYPE_Q6_K:
             return true;
         default:
@@ -3584,54 +3536,6 @@ static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, d
     return true;
 }
 
-static bool reorder_qw_q5_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {
-    GGML_ASSERT(size % sizeof(block_q5_K) == 0);
-    GGML_ASSERT(offset % sizeof(block_q5_K) == 0);
-
-    const int nblocks = size / sizeof(block_q5_K);
-
-    sycl_reorder_temp_buffer tmp(stream, size);
-    if (!tmp) {
-        GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size);
-        return false;
-    }
-    uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);
-
-    sycl::event copy_event;
-    SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size)));
-    if (!g_ggml_sycl_use_async_mem_op) {
-        copy_event.wait();
-    }
-
-    auto * qs_ptr     = data_device;
-    auto * qh_ptr     = qs_ptr + (QK_K / 2) * nblocks;
-    auto * scales_ptr = qh_ptr + (QK_K / 8) * nblocks;
-    auto * dm_ptr     = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * nblocks);
-
-    auto reorder_event = stream->parallel_for(nblocks, [=](auto i) {
-        const block_q5_K * x  = (const block_q5_K *) tmp_buf;
-        const int          ib = i;
-
-        for (int j = 0; j < QK_K / 2; ++j) {
-            qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j];
-        }
-
-        for (int j = 0; j < QK_K / 8; ++j) {
-            qh_ptr[ib * (QK_K / 8) + j] = x[ib].qh[j];
-        }
-
-        for (int j = 0; j < K_SCALE_SIZE; ++j) {
-            scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j];
-        }
-
-        dm_ptr[ib] = x[ib].dm;
-    });
-    if (!g_ggml_sycl_use_async_mem_op) {
-        reorder_event.wait_and_throw();
-    }
-    return true;
-}
-
 static bool reorder_qw_q6_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {
     GGML_ASSERT(size % sizeof(block_q6_K) == 0);
     GGML_ASSERT(offset % sizeof(block_q6_K) == 0);
@@ -3698,8 +3602,6 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
             return reorder_qw_q8_0(data_device, ncols, nrows, size, 0, stream);
         case GGML_TYPE_Q4_K:
             return reorder_qw_q4_k(data_device, size, 0, stream);
-        case GGML_TYPE_Q5_K:
-            return reorder_qw_q5_k(data_device, size, 0, stream);
         case GGML_TYPE_Q6_K:
             return reorder_qw_q6_k(data_device, size, 0, stream);
         default:
@@ -4492,21 +4394,6 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg
         case GGML_OP_SSM_CONV:
             ggml_sycl_ssm_conv(ctx, dst);
             break;
-        case GGML_OP_SSM_SCAN:
-            ggml_sycl_ssm_scan(ctx, dst);
-            break;
-        case GGML_OP_FILL:
-            ggml_sycl_fill(ctx, dst);
-            break;
-        case GGML_OP_CUMSUM:
-            ggml_sycl_cumsum(ctx, dst);
-            break;
-        case GGML_OP_DIAG:
-            ggml_sycl_diag(ctx, dst);
-            break;
-        case GGML_OP_SOLVE_TRI:
-            ggml_sycl_solve_tri(ctx, dst);
-            break;
         case GGML_OP_ROLL:
             ggml_sycl_roll(ctx, dst);
             break;
@@ -5015,7 +4902,6 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g
             {
                 switch (op->src[0]->type) {
                     case GGML_TYPE_F16:
-                    case GGML_TYPE_BF16:
                     case GGML_TYPE_F32:
                     case GGML_TYPE_Q4_0:
                     case GGML_TYPE_Q4_1:
@@ -5198,10 +5084,11 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g
         case GGML_OP_ACC:
             return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
         case GGML_OP_PAD:
+            // TODO: add circular padding support for syscl, see https://github.com/ggml-org/llama.cpp/pull/16985
             if (ggml_get_op_params_i32(op, 8) != 0) {
                 return false;
             }
-            return true;
+            return ggml_is_contiguous(op->src[0]);
         case GGML_OP_LEAKY_RELU:
         case GGML_OP_TIMESTEP_EMBEDDING:
         case GGML_OP_RWKV_WKV6:
@@ -5217,21 +5104,6 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g
             return op->type == GGML_TYPE_F32;
         case GGML_OP_ARANGE:
             return op->type == GGML_TYPE_F32;
-        case GGML_OP_SSM_SCAN:
-            if (op->src[3]->ne[0] == 1) {
-                // Mamba2
-                // (kernel only supports (d_state == 128 || d_state == 256) && d_head % WARP_SIZE == 0)
-                return (op->src[0]->ne[0] == 128 || op->src[0]->ne[0] == 256) && op->src[0]->ne[1] % WARP_SIZE == 0;
-            } else {
-                // TODO Mamba-1 not yet ported to SYCL
-                return false;
-            }
-        case GGML_OP_FILL:
-        case GGML_OP_CUMSUM:
-        case GGML_OP_DIAG:
-            return true;
-        case GGML_OP_SOLVE_TRI:
-            return op->src[0]->ne[0] <= SYCL_SOLVE_TRI_MAX_N && op->src[1]->ne[0] <= SYCL_SOLVE_TRI_MAX_K;
         case GGML_OP_FLASH_ATTN_EXT:
             return ggml_sycl_flash_attn_ext_supported(device, op);
         default:

ggml/src/ggml-sycl/mmvq.cpp

@@ -839,26 +839,6 @@ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy,
     }
 }
 
-static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
-                                               const int nrows, dpct::queue_ptr stream) {
-    GGML_ASSERT(ncols % QK_K == 0);
-
-    const int        block_num_y   = ceil_div(nrows, GGML_SYCL_MMV_Y);
-    constexpr size_t num_subgroups = 16;
-    GGML_ASSERT(block_num_y % num_subgroups == 0);
-
-    const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE);
-    const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
-
-    stream->submit([&](sycl::handler & cgh) {
-        cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size),
-                            [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                                mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>>(vx, vy, dst, ncols,
-                                                                                            nrows, nd_item);
-                            });
-    });
-}
-
 static void reorder_mul_mat_vec_q6_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
                                                const int nrows, dpct::queue_ptr stream) {
     GGML_ASSERT(ncols % QK_K == 0);
@@ -1145,7 +1125,6 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                     GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q8_0_q8_1_sycl\n");
                     reorder_mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
                 } else {
-                    GGML_SYCL_DEBUG("Calling mul_mat_vec_q8_0_q8_1_sycl\n");
                     mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
                 }
                 break;
@@ -1166,14 +1145,7 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                 }
                 break;
             case GGML_TYPE_Q5_K:
-                if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
-                    ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                    GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q5_k_q8_1_sycl\n");
-                    reorder_mul_mat_vec_q5_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
-                } else {
-                    GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_K_q8_1_sycl\n");
-                    mul_mat_vec_q5_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
-                }
+                mul_mat_vec_q5_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
                 break;
             case GGML_TYPE_Q6_K:
                 if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&

ggml/src/ggml-sycl/pad.cpp

@@ -13,8 +13,7 @@
 //#include "common.hpp"
 #include "pad.hpp"
 
-static void pad_f32(const float * src, size_t s00, size_t s01, size_t s02, size_t s03,
-                    float * dst,
+static void pad_f32(const float * src, float * dst,
                     const int lp0, const int rp0, const int lp1, const int rp1,
                     const int lp2, const int rp2, const int lp3, const int rp3,
                     const int ne0, const int ne1, const int ne2, const int ne3,
@@ -28,6 +27,7 @@ static void pad_f32(const float * src, size_t s00, size_t s01, size_t s02, size_
         return;
     }
 
+    // operation
     const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;
     if ((i0 >= lp0 && i0 < ne0 - rp0) &&
         (i1 >= lp1 && i1 < ne1 - rp1) &&
@@ -37,8 +37,12 @@ static void pad_f32(const float * src, size_t s00, size_t s01, size_t s02, size_
         const int64_t i01 = i1 - lp1;
         const int64_t i02 = i2 - lp2;
         const int64_t i03 = i3 - lp3;
+        const int64_t ne02 = ne2 - lp2 - rp2;
+        const int64_t ne01 = ne1 - lp1 - rp1;
+        const int64_t ne00 = ne0 - lp0 - rp0;
 
-        const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00;
+        const int64_t src_idx = i03 * (ne00 * ne01 * ne02) +
+                                i02 * (ne00 * ne01) + i01 * ne00 + i00;
 
         dst[dst_idx] = src[src_idx];
     } else {
@@ -46,19 +50,20 @@ static void pad_f32(const float * src, size_t s00, size_t s01, size_t s02, size_
     }
 }
 
-static void pad_f32_sycl(const float * src, size_t s00, size_t s01, size_t s02, size_t s03,
-                         float * dst, const int lp0, const int rp0, const int lp1, const int rp1,
-                         const int lp2, const int rp2, const int lp3, const int rp3,
-                         const int ne0, const int ne1, const int ne2, const int ne3,
+static void pad_f32_sycl(const float *src, float *dst, const int lp0,
+                         const int rp0, const int lp1, const int rp1,
+                         const int lp2, const int rp2, const int lp3,
+                         const int rp3, const int ne0, const int ne1,
+                         const int ne2, const int ne3,
                          dpct::queue_ptr stream) {
     int num_blocks = (ne0 + SYCL_PAD_BLOCK_SIZE - 1) / SYCL_PAD_BLOCK_SIZE;
-    sycl::range<3> grid(ne2 * ne3, ne1, num_blocks);
+    dpct::dim3 gridDim(num_blocks, ne1, ne2 * ne3);
     stream->parallel_for(
-        sycl::nd_range<3>(grid * sycl::range<3>(1, 1, SYCL_PAD_BLOCK_SIZE),
+        sycl::nd_range<3>(gridDim * sycl::range<3>(1, 1, SYCL_PAD_BLOCK_SIZE),
                           sycl::range<3>(1, 1, SYCL_PAD_BLOCK_SIZE)),
         [=](sycl::nd_item<3> item_ct1) {
-            pad_f32(src, s00, s01, s02, s03, dst, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3,
-                    ne0, ne1, ne2, ne3, item_ct1);
+            pad_f32(src, dst, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3, ne0, ne1,
+                    ne2, ne3, item_ct1);
         });
 }
 
@@ -66,27 +71,22 @@ void ggml_sycl_op_pad(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     const ggml_tensor * src0 = dst->src[0];
     const float * src0_d = (const float *)src0->data;
     float * dst_d = (float *)dst->data;
-    dpct::queue_ptr stream = ctx.stream();
+    dpct::queue_ptr     stream = ctx.stream();
 
     GGML_ASSERT(src0->type == GGML_TYPE_F32);
     GGML_ASSERT(dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(ggml_is_contiguous(src0));
 
-    const size_t ts = ggml_type_size(src0->type);
-    const size_t s00 = src0->nb[0] / ts;
-    const size_t s01 = src0->nb[1] / ts;
-    const size_t s02 = src0->nb[2] / ts;
-    const size_t s03 = src0->nb[3] / ts;
+    const int32_t lp0 = ((const int32_t*)(dst->op_params))[0];
+    const int32_t rp0 = ((const int32_t*)(dst->op_params))[1];
+    const int32_t lp1 = ((const int32_t*)(dst->op_params))[2];
+    const int32_t rp1 = ((const int32_t*)(dst->op_params))[3];
+    const int32_t lp2 = ((const int32_t*)(dst->op_params))[4];
+    const int32_t rp2 = ((const int32_t*)(dst->op_params))[5];
+    const int32_t lp3 = ((const int32_t*)(dst->op_params))[6];
+    const int32_t rp3 = ((const int32_t*)(dst->op_params))[7];
 
-    const int32_t lp0 = ((const int32_t *)(dst->op_params))[0];
-    const int32_t rp0 = ((const int32_t *)(dst->op_params))[1];
-    const int32_t lp1 = ((const int32_t *)(dst->op_params))[2];
-    const int32_t rp1 = ((const int32_t *)(dst->op_params))[3];
-    const int32_t lp2 = ((const int32_t *)(dst->op_params))[4];
-    const int32_t rp2 = ((const int32_t *)(dst->op_params))[5];
-    const int32_t lp3 = ((const int32_t *)(dst->op_params))[6];
-    const int32_t rp3 = ((const int32_t *)(dst->op_params))[7];
-
-    pad_f32_sycl(src0_d, s00, s01, s02, s03, dst_d,
+    pad_f32_sycl(src0_d, dst_d,
                  lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3,
                  dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], stream);
 }

ggml/src/ggml-sycl/quants.hpp

@@ -79,31 +79,6 @@ template <> struct block_q_t<GGML_TYPE_Q4_K> {
     static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
 };
 
-template <> struct block_q_t<GGML_TYPE_Q5_K> {
-    struct traits {
-        static constexpr uint32_t qk       = QK_K;
-        static constexpr uint32_t qi       = QI5_K;
-        static constexpr uint32_t qr       = QR5_K;
-        static constexpr uint32_t vdr_mmvq = 2;
-    };
-
-    // Reordered layout: [qs (QK_K/2 per block)] [qh (QK_K/8 per block)] [scales] [dm]
-    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
-        auto qs_offset = block_index * (QK_K / 2);
-        auto qh_offset = n_blocks * (QK_K / 2) + block_index * (QK_K / 8);
-        return { qs_offset, qh_offset };
-    }
-
-    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
-        auto nblocks        = (nrows * (ncols / QK_K));
-        auto total_qs_bytes = nblocks * (QK_K / 2) + nblocks * (QK_K / 8);
-        return { total_qs_bytes + block_index * K_SCALE_SIZE,
-                 total_qs_bytes + nblocks * K_SCALE_SIZE + block_index * sizeof(ggml_half2) };
-    }
-
-    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
-};
-
 template <> struct block_q_t<GGML_TYPE_Q6_K> {
     struct traits {
         static constexpr uint32_t qk       = QK_K;

ggml/src/ggml-sycl/solve_tri.cpp

@@ -1,172 +0,0 @@
-#include "solve_tri.hpp"
-#include "common.hpp"
-#include <oneapi/mkl/blas.hpp>
-
-template <int n_template, int k_template>
-static void solve_tri_f32_fast(const float * __restrict__ A,
-                               const float * __restrict__ B,
-                               float * __restrict__ X,
-                               const int64_t ne02, [[maybe_unused]] const int64_t ne03,
-                               const int64_t nb02, const int64_t nb03,
-                               const int64_t nb12, const int64_t nb13,
-                               const int64_t nb2,  const int64_t nb3,
-                               const int n_arg, const int k_arg,
-                               const sycl::nd_item<2> & item, float * sA) {
-
-    const int n = n_template == 0 ? n_arg : n_template;
-    const int k = k_template == 0 ? k_arg : k_template;
-
-    const int batch_idx = item.get_group(1);
-    const int lane      = item.get_local_id(1) % WARP_SIZE;
-    const int col_idx   = item.get_local_id(0);
-
-    if (col_idx >= k) {
-        return;
-    }
-
-    const int64_t i03 = batch_idx / ne02;
-    const int64_t i02 = batch_idx % ne02;
-
-    const float * A_batch = (const float *) ((const char *) A + i02 * nb02 + i03 * nb03);
-    const float * B_batch = (const float *) ((const char *) B + i02 * nb12 + i03 * nb13);
-    float *       X_batch = (float *)       ((char *)       X + i02 * nb2  + i03 * nb3);
-
-    const int offset = item.get_local_id(1) + item.get_local_id(0) * item.get_local_range(1);
-
-#pragma unroll
-    for (int i = 0; i < n * n; i += k * WARP_SIZE) {
-        const int i0 = i + offset;
-        if (i0 < n * n) {
-            sA[i0] = A_batch[i0];
-        }
-    }
-
-    item.barrier(sycl::access::fence_space::local_space);
-
-    float x_low  = (lane < n) ? B_batch[lane * k + col_idx] : 0.0f;
-    float x_high = (WARP_SIZE + lane < n) ? B_batch[(WARP_SIZE + lane) * k + col_idx] : 0.0f;
-
-    const int half      = WARP_SIZE;
-    const int nrows_low = (n < half) ? n : half;
-
-#pragma unroll
-    for (int row = 0; row < nrows_low; ++row) {
-        float sum = 0.0f;
-        if (lane < row) {
-            sum += sA[row * n + lane] * x_low;
-        }
-        sum = warp_reduce_sum<WARP_SIZE>(sum);
-        if (lane == row) {
-            x_low = (x_low - sum) / sA[row * n + row];
-        }
-    }
-
-#pragma unroll
-    for (int row = half; row < n; ++row) {
-        float     sum = sA[row * n + lane] * x_low;
-        const int j   = half + lane;
-        if (j < row) {
-            sum += sA[row * n + j] * x_high;
-        }
-        sum = warp_reduce_sum<WARP_SIZE>(sum);
-        if (lane == row - half) {
-            x_high = (x_high - sum) / sA[row * n + row];
-        }
-    }
-
-#pragma unroll
-    for (int rr = 0; rr < 2; ++rr) {
-        const int row = rr * WARP_SIZE + lane;
-        if (row < n) {
-            const float val            = (row < half) ? x_low : x_high;
-            X_batch[row * k + col_idx] = val;
-        }
-    }
-}
-
-static void solve_tri_f32_mkl(dpct::queue_ptr stream,
-                               const float * A, float * X,
-                               int n, int k,
-                               int64_t ne02, [[maybe_unused]] int64_t ne03,
-                               int64_t nb02, [[maybe_unused]] int64_t nb03,
-                               int64_t nb2,  [[maybe_unused]] int64_t nb3) {
-    const float alpha = 1.0f;
-    const int64_t total_batches = ne02 * ne03;
-    if (total_batches == 0) {
-        return;
-    }
-
-    const int64_t stride_a = nb02 / sizeof(float);
-    const int64_t stride_x = nb2 / sizeof(float);
-
-    oneapi::mkl::blas::trsm_batch(
-        *stream,
-        oneapi::mkl::side::right,
-        oneapi::mkl::uplo::upper,
-        oneapi::mkl::transpose::nontrans,
-        oneapi::mkl::diag::nonunit,
-        k, n, alpha,
-        A, n, stride_a,
-        X, k, stride_x,
-        total_batches);
-}
-
-inline void ggml_sycl_op_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    const ggml_tensor * src0 = dst->src[0];
-    const ggml_tensor * src1 = dst->src[1];
-
-    GGML_ASSERT(ggml_is_contiguous(src0));
-    GGML_ASSERT(ggml_is_contiguous(src1));
-    GGML_ASSERT(src0->type == GGML_TYPE_F32);
-
-    dpct::queue_ptr stream = ctx.stream();
-    SYCL_CHECK(ggml_sycl_set_device(ctx.device));
-
-    const int n    = src0->ne[0];
-    const int k    = src1->ne[0];
-    const int64_t ne02 = src0->ne[2];
-    const int64_t ne03 = src0->ne[3];
-
-    GGML_ASSERT(n <= SYCL_SOLVE_TRI_MAX_N && k <= SYCL_SOLVE_TRI_MAX_K);
-
-    const float * A_d = static_cast<const float *>(src0->data);
-    const float * B_d = static_cast<const float *>(src1->data);
-    float * X_d       = static_cast<float *>(dst->data);
-
-    if (X_d != B_d) {
-        const int64_t total_elements = (int64_t)n * k * ne02 * ne03;
-        stream->memcpy(X_d, B_d, total_elements * sizeof(float));
-    }
-
-    const int64_t nb02 = src0->nb[2];
-    const int64_t nb03 = src0->nb[3];
-    const int64_t nb12 = src1->nb[2];
-    const int64_t nb13 = src1->nb[3];
-    const int64_t nb2  = dst->nb[2];
-    const int64_t nb3  = dst->nb[3];
-
-    const int64_t total_batches = ne02 * ne03;
-
-    if (n <= 2 * WARP_SIZE && k <= 32) {
-        const int smem_size = 2 * WARP_SIZE * 2 * WARP_SIZE;
-        const sycl::range<2> grid(1, total_batches);
-        const sycl::range<2> block(k, WARP_SIZE);
-        stream->submit([&](sycl::handler & cgh) {
-            sycl::local_accessor<float, 1> smem_acc(sycl::range<1>(smem_size), cgh);
-            cgh.parallel_for(
-                sycl::nd_range<2>(grid * block, block),
-                [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                    solve_tri_f32_fast<0, 0>(A_d, B_d, X_d, ne02, ne03,
-                                              nb02, nb03, nb12, nb13, nb2, nb3,
-                                              n, k, item, get_pointer(smem_acc));
-                });
-        });
-    } else {
-        solve_tri_f32_mkl(stream, A_d, X_d, n, k, ne02, ne03, nb02, nb03, nb2, nb3);
-    }
-}
-
-void ggml_sycl_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
-    ggml_sycl_op_solve_tri(ctx, dst);
-}

ggml/src/ggml-sycl/solve_tri.hpp

@@ -1,8 +0,0 @@
-#pragma once
-
-#include "common.hpp"
-
-#define SYCL_SOLVE_TRI_MAX_N 64
-#define SYCL_SOLVE_TRI_MAX_K 64
-
-void ggml_sycl_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst);

ggml/src/ggml-sycl/ssm_conv.cpp

@@ -63,7 +63,7 @@ static void kernel_ssm_conv(
     });
 }
 
-inline void ggml_sycl_op_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+void ggml_sycl_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     ggml_tensor * src0 = dst->src[0];
     ggml_tensor * src1 = dst->src[1];
 
@@ -125,8 +125,3 @@ inline void ggml_sycl_op_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor *
         throw;
     }
 }
-
-void ggml_sycl_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
-    ggml_sycl_op_ssm_conv(ctx, dst);
-}

ggml/src/ggml-sycl/ssm_scan.cpp

@@ -1,156 +0,0 @@
-#include "ssm_scan.hpp"
-#include "common.hpp"
-
-template <int c_factor, int d_state>
-static void ssm_scan_f32_group(
-        const float * __restrict__ src0, const float * __restrict__ src1, const float * __restrict__ src2,
-        const float * __restrict__ src3, const float * __restrict__ src4, const float * __restrict__ src5,
-        const int32_t * __restrict__ src6, float * __restrict__ dst,
-        const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3,
-        const int src2_nb1, const int src2_nb2, const int src3_nb1,
-        const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3,
-        const int64_t s_off, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok,
-        const sycl::nd_item<2> & item) {
-
-    const int lane     = item.get_local_id(1) % WARP_SIZE;
-    const int warp     = item.get_local_id(1) / WARP_SIZE;
-    const int warp_idx = item.get_group(1) * c_factor + warp;
-    const int seq_idx  = item.get_group(0);
-
-    const int head_idx = warp_idx / d_head;
-    const int head_off = (warp_idx % d_head) * sizeof(float);
-    const int group_off = (head_idx / (n_head / n_group)) * d_state * sizeof(float);
-
-    const float * s0_warp = (const float *) ((const char *) src0 + src6[seq_idx] * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
-    const float * x_warp  = (const float *) ((const char *) src1 + (seq_idx * src1_nb3) + (warp_idx * sizeof(float)));
-    const float * dt_warp = (const float *) ((const char *) src2 + (seq_idx * src2_nb2) + head_idx * sizeof(float));
-    const float * A_warp  = (const float *) ((const char *) src3 + head_idx * src3_nb1);
-    const float * B_warp  = (const float *) ((const char *) src4 + (seq_idx * src4_nb3) + (group_off));
-    const float * C_warp  = (const float *) ((const char *) src5 + (seq_idx * src5_nb3) + (group_off));
-    float *       y_warp  = dst + (seq_idx * n_tok * n_head * d_head) + warp_idx;
-    float *       s_warp  = (float *) ((char *) dst + s_off + seq_idx * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
-
-    const int stride_x  = src1_nb2 / sizeof(float);
-    const int stride_dt = src2_nb1 / sizeof(float);
-    const int stride_B  = src4_nb2 / sizeof(float);
-    const int stride_C  = src5_nb2 / sizeof(float);
-    const int stride_y  = n_head * d_head;
-
-    float state[c_factor];
-    float state_sum = 0.0f;
-
-#pragma unroll
-    for (int j = 0; j < c_factor; j++) {
-        state[j] = s0_warp[WARP_SIZE * j + lane];
-    }
-
-    for (int64_t i = 0; i < n_tok; i++) {
-        const float dt_val = dt_warp[i * stride_dt];
-        const float dt_soft_plus = (dt_val <= 20.0f ? sycl::log1p(sycl::exp(dt_val)) : dt_val);
-
-        state_sum = 0.0f;
-        const float dA   = sycl::exp(dt_soft_plus * A_warp[0]);
-        const float x_dt = x_warp[i * stride_x] * dt_soft_plus;
-#pragma unroll
-        for (int j = 0; j < c_factor; j++) {
-            const float B_val = B_warp[i * stride_B + WARP_SIZE * j + lane];
-            const float C_val = C_warp[i * stride_C + WARP_SIZE * j + lane];
-            state[j] = (state[j] * dA) + (B_val * x_dt);
-            state_sum += state[j] * C_val;
-        }
-
-        state_sum = warp_reduce_sum<WARP_SIZE>(state_sum);
-
-        if (lane == 0) {
-            y_warp[i * stride_y] = state_sum;
-        }
-    }
-
-#pragma unroll
-    for (int j = 0; j < c_factor; j++) {
-        s_warp[WARP_SIZE * j + lane] = state[j];
-    }
-}
-
-static void ssm_scan_f32_sycl(
-        const float * src0, const float * src1, const float * src2, const float * src3,
-        const float * src4, const float * src5, const int32_t * src6, float * dst,
-        const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3, const int src2_nb1,
-        const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2,
-        const int src5_nb3, const int64_t s_off, const int64_t d_state, const int64_t head_dim,
-        const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq,
-        dpct::queue_ptr stream) {
-
-    // NOTE: if you change conditions here, be sure to update the corresponding supports_op condition!
-    GGML_ASSERT(src3_nb1 == sizeof(float));
-    if (d_state == 128) {
-        constexpr int threads   = 128;
-        constexpr int num_warps = threads / WARP_SIZE;
-        const sycl::range<2> grid(n_seq, (n_head * head_dim + num_warps - 1) / num_warps);
-        const sycl::range<2> block(1, threads);
-        stream->parallel_for(
-            sycl::nd_range<2>(grid * block, block),
-            [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                ssm_scan_f32_group<128 / WARP_SIZE, 128>(
-                    src0, src1, src2, src3, src4, src5, src6, dst,
-                    src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,
-                    src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, item);
-            });
-    } else if (d_state == 256) {
-        constexpr int threads   = 256;
-        constexpr int num_warps = threads / WARP_SIZE;
-        const sycl::range<2> grid(n_seq, (n_head * head_dim + num_warps - 1) / num_warps);
-        const sycl::range<2> block(1, threads);
-        stream->parallel_for(
-            sycl::nd_range<2>(grid * block, block),
-            [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                ssm_scan_f32_group<256 / WARP_SIZE, 256>(
-                    src0, src1, src2, src3, src4, src5, src6, dst,
-                    src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,
-                    src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, item);
-            });
-    } else {
-        GGML_ABORT("ssm_scan: unsupported d_state (must be 128 or 256)");
-    }
-}
-
-inline void ggml_sycl_op_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    const ggml_tensor * src0 = dst->src[0];
-    const ggml_tensor * src1 = dst->src[1];
-    const ggml_tensor * src2 = dst->src[2];
-    const ggml_tensor * src3 = dst->src[3];
-    const ggml_tensor * src4 = dst->src[4];
-    const ggml_tensor * src5 = dst->src[5];
-    const ggml_tensor * src6 = dst->src[6];
-
-    GGML_ASSERT(src0->type == GGML_TYPE_F32);
-    GGML_ASSERT(src6->type == GGML_TYPE_I32);
-    GGML_ASSERT(dst->type  == GGML_TYPE_F32);
-
-    const int64_t nc  = src0->ne[0];
-    const int64_t nr  = src0->ne[1];
-    const int64_t nh  = src1->ne[1];
-    const int64_t ng  = src4->ne[1];
-    const int64_t n_t = src1->ne[2];
-    const int64_t n_s = src1->ne[3];
-    const int64_t s_off = ggml_nelements(src1) * sizeof(float);
-
-    GGML_ASSERT(ggml_nelements(src1) + nc * nr * nh * n_s == ggml_nelements(dst));
-
-    dpct::queue_ptr stream = ctx.stream();
-    SYCL_CHECK(ggml_sycl_set_device(ctx.device));
-
-    ssm_scan_f32_sycl(
-        static_cast<const float *>(src0->data), static_cast<const float *>(src1->data),
-        static_cast<const float *>(src2->data), static_cast<const float *>(src3->data),
-        static_cast<const float *>(src4->data), static_cast<const float *>(src5->data),
-        static_cast<const int32_t *>(src6->data), static_cast<float *>(dst->data),
-        src0->nb[2], src0->nb[3], src1->nb[2], src1->nb[3], src2->nb[1], src2->nb[2],
-        src3->nb[1], src4->nb[2], src4->nb[3], src5->nb[2], src5->nb[3],
-        s_off, nc, nr, nh, ng, n_t, n_s, stream);
-}
-
-void ggml_sycl_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/7);
-    ggml_sycl_op_ssm_scan(ctx, dst);
-}

ggml/src/ggml-sycl/ssm_scan.hpp

@@ -1,5 +0,0 @@
-#pragma once
-
-#include "common.hpp"
-
-void ggml_sycl_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst);

ggml/src/ggml-sycl/vecdotq.hpp

@@ -357,31 +357,38 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
     using q8_0_block  = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q8_0>;
     using q8_0_traits = typename q8_0_block::traits;
 
-    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
-                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
-                                     const sycl::half2 * q8_1_ds, const int & iqs) {
-        const uint8_t * base = static_cast<const uint8_t *>(vbq);
-        const int8_t *  qs   = reinterpret_cast<const int8_t *>(base + ibx_offset.first);
-        const ggml_half  d   = *reinterpret_cast<const ggml_half *>(base + d_offset.first);
-
-        int v[q8_0_traits::vdr_mmvq];
-        int u[q8_0_traits::vdr_mmvq];
-
-#pragma unroll
-        for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
-            v[i] = get_int_from_int8(qs, iqs + i);
-            u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);
-        }
-
+    __dpct_inline__ float vec_dot_q8_0_q8_1_impl(const int * v, const int * u, const float & d8_0, const sycl::half2 & ds8) {
         int sumi = 0;
+
 #pragma unroll
         for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
+            // Q8_0 values are signed int8, no nibble extraction needed
+            // Direct dp4a: each int packs 4 int8 values
             sumi = dpct::dp4a(v[i], u[i], sumi);
         }
 
-        const sycl::half2 ds_values = *q8_1_ds;
-        return static_cast<float>(d) * static_cast<float>(ds_values[0]) * sumi;
+        const sycl::float2 ds8f = ds8.convert<float, sycl::rounding_mode::automatic>();
+
+        // Q8_0 has no bias term (values are signed), so just scale
+        return d8_0 * sumi * ds8f.x();
     }
+
+    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
+                                     const sycl::half2 * q8_1_ds, const int & iqs) {
+        const int8_t * bq8_0 = static_cast<const int8_t *>(vbq) + ibx_offset.first;
+        const ggml_half d = *(reinterpret_cast<const ggml_half *>(static_cast<const uint8_t *>(vbq) + d_offset.first));
+        int             v[q8_0_traits::vdr_mmvq];
+        int             u[q8_0_traits::vdr_mmvq];
+
+#pragma unroll
+        for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
+            v[i] = get_int_from_int8(bq8_0, iqs + i);
+            u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);
+        }
+
+        return vec_dot_q8_0_q8_1_impl(v, u, d, *q8_1_ds);
+    };
 };
 
 static inline float vec_dot_q4_K_q8_1_common(const int * __restrict__ q4, const uint16_t * __restrict__ scales,
@@ -474,65 +481,6 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K> {
     }
 };
 
-template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K> {
-    static constexpr ggml_type gtype = GGML_TYPE_Q5_K;
-
-    using q5_k_block  = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q5_K>;
-    using q5_k_traits = typename q5_k_block::traits;
-
-    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
-                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
-                                     const sycl::half2 * q8_1_ds, const int & iqs) {
-        const uint8_t *    base           = static_cast<const uint8_t *>(vbq);
-        const uint8_t *    qs             = base + ibx_offset.first;   // low 4 bits
-        const uint8_t *    qh_base        = base + ibx_offset.second;  // high bit
-        const uint8_t *    scs            = base + d_offset.first;
-        const ggml_half2 * dms            = reinterpret_cast<const ggml_half2 *>(base + d_offset.second);
-
-        const int        bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2));
-        const int *      ql_ptr     = (const int *) (qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));
-        const int *      qh_ptr     = (const int *) (qh_base + 4 * ((iqs / 2) % 4));
-        const uint16_t * scales     = (const uint16_t *) scs;
-
-        int   vl[2];
-        int   vh[2];
-        int   u[2 * QR5_K];
-        float d8[QR5_K];
-
-        vl[0] = ql_ptr[0];
-        vl[1] = ql_ptr[4];
-
-        vh[0] = qh_ptr[0] >> bq8_offset;
-        vh[1] = qh_ptr[4] >> bq8_offset;
-
-        uint16_t  aux[2];
-        const int j = (QR5_K * ((iqs / 2) / (QI8_1 / 2))) / 2;
-        if (j < 2) {
-            aux[0] = scales[j + 0] & 0x3f3f;
-            aux[1] = scales[j + 2] & 0x3f3f;
-        } else {
-            aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);
-            aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);
-        }
-
-        const uint8_t * sc = (const uint8_t *) aux;
-        const uint8_t * m  = sc + 2;
-
-        for (int i = 0; i < QR5_K; ++i) {
-            const int8_t* quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1;
-            sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i);
-
-            d8[i]                   = ds_values[0];
-
-            const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4);
-            u[2 * i + 0]   = q8[0];
-            u[2 * i + 1]   = q8[4];
-        }
-
-        return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, *dms, d8);
-    }
-};
-
 template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K> {
     static constexpr ggml_type gtype = GGML_TYPE_Q6_K;
 

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -2149,11 +2149,11 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
 
     // Patch SPIR-V to enable RTE rounding for FP16, avoiding the need for
     // separate shader variants compiled with -DRTE16.
-    std::vector<uint32_t> spirv;
+    std::vector<uint32_t> spv;
     if (device->float_controls_rte_fp16) {
         const uint32_t* spv_words = reinterpret_cast<const uint32_t *>(spv_data);
         size_t word_count = spv_size / sizeof(uint32_t);
-        spirv.assign(spv_words, spv_words + word_count);
+        spv.assign(spv_words, spv_words + word_count);
 
         // Find insertion points respecting SPIR-V layout order:
         //   Header(5) -> OpCapability -> OpExtension -> ... -> OpEntryPoint -> OpExecutionMode -> ...
@@ -2163,9 +2163,9 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
         size_t exec_insert_pos = pos;
         uint32_t entry_point_id = 0;
 
-        while (pos < spirv.size()) {
-            uint32_t opcode = spirv[pos] & spv::OpCodeMask;
-            uint32_t len    = spirv[pos] >> spv::WordCountShift;
+        while (pos < spv.size()) {
+            uint32_t opcode = spv[pos] & spv::OpCodeMask;
+            uint32_t len    = spv[pos] >> spv::WordCountShift;
             if (len == 0) break;
 
             if (opcode == spv::OpCapability) {
@@ -2174,7 +2174,7 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
             } else if (opcode == spv::OpExtension) {
                 ext_insert_pos = pos + len;
             } else if (opcode == spv::OpEntryPoint) {
-                entry_point_id = spirv[pos + 2];
+                entry_point_id = spv[pos + 2];
                 exec_insert_pos = pos + len;
             } else if (opcode == spv::OpExecutionMode || opcode == spv::OpExecutionModeId) {
                 exec_insert_pos = pos + len;
@@ -2189,7 +2189,7 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
 
         // OpExecutionMode %entrypoint RoundingModeRTE 16
         uint32_t exec_mode[] = { (4u << spv::WordCountShift) | spv::OpExecutionMode, entry_point_id, spv::ExecutionModeRoundingModeRTE, 16 };
-        spirv.insert(spirv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode));
+        spv.insert(spv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode));
 
         // OpExtension "SPV_KHR_float_controls"
         const char ext_str[] = "SPV_KHR_float_controls";
@@ -2197,13 +2197,13 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
         std::vector<uint32_t> extension(1 + ext_str_words, 0);
         extension[0] = (uint32_t)((1 + ext_str_words) << spv::WordCountShift) | spv::OpExtension;
         memcpy(&extension[1], ext_str, sizeof(ext_str));
-        spirv.insert(spirv.begin() + ext_insert_pos, extension.begin(), extension.end());
+        spv.insert(spv.begin() + ext_insert_pos, extension.begin(), extension.end());
 
         // OpCapability RoundingModeRTE
         uint32_t capability[] = { (2u << spv::WordCountShift) | spv::OpCapability, spv::CapabilityRoundingModeRTE };
-        spirv.insert(spirv.begin() + cap_insert_pos, std::begin(capability), std::end(capability));
+        spv.insert(spv.begin() + cap_insert_pos, std::begin(capability), std::end(capability));
 
-        shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data());
+        shader_module_create_info = vk::ShaderModuleCreateInfo({}, spv.size() * sizeof(uint32_t), spv.data());
     }
 
     pipeline->shader_module = device->device.createShaderModule(shader_module_create_info);

gguf-py/gguf/constants.py

@@ -152,6 +152,9 @@ class Keys:
         SWIGLU_CLAMP_SHEXP                = "{arch}.swiglu_clamp_shexp"
         DENSE_FEAT_IN_SIZE                = "{arch}.{dense}_feat_in"
         DENSE_FEAT_OUT_SIZE               = "{arch}.{dense}_feat_out"
+        EAGLE3_EXTRACT_LAYERS             = "{arch}.extract_layers"
+        EAGLE3_TARGET_HIDDEN_SIZE         = "{arch}.target_hidden_size"
+        EAGLE3_NORM_BEFORE_RESIDUAL       = "{arch}.norm_before_residual"
 
     class Attention:
         HEAD_COUNT                   = "{arch}.attention.head_count"
@@ -175,7 +178,6 @@ class Keys:
         SLIDING_WINDOW               = "{arch}.attention.sliding_window"
         SCALE                        = "{arch}.attention.scale"
         OUTPUT_SCALE                 = "{arch}.attention.output_scale"
-        VALUE_SCALE                  = "{arch}.attention.value_scale"
         TEMPERATURE_LENGTH           = "{arch}.attention.temperature_length"
         KEY_LENGTH_MLA               = "{arch}.attention.key_length_mla"
         VALUE_LENGTH_MLA             = "{arch}.attention.value_length_mla"
@@ -340,9 +342,6 @@ class Keys:
         FEED_FORWARD_LENGTH = "clip.audio.feed_forward_length"
         PROJECTION_DIM      = "clip.audio.projection_dim"
         BLOCK_COUNT         = "clip.audio.block_count"
-        CHUNK_SIZE          = "clip.audio.chunk_size"
-        CONV_KERNEL_SIZE    = "clip.audio.conv_kernel_size"
-        MAX_POS_EMB         = "clip.audio.max_pos_emb"
 
         class Attention:
             HEAD_COUNT      = "clip.audio.attention.head_count"
@@ -350,9 +349,6 @@ class Keys:
 
         class Projector:
             STACK_FACTOR    = "clip.audio.projector.stack_factor"
-            WINDOW_SIZE     = "clip.audio.projector.window_size"
-            DOWNSAMPLE_RATE = "clip.audio.projector.downsample_rate"
-            HEAD_COUNT      = "clip.audio.projector.head_count"
 
     class Diffusion:
         SHIFT_LOGITS        = "diffusion.shift_logits"
@@ -496,6 +492,7 @@ class MODEL_ARCH(IntEnum):
     RND1             = auto()
     PANGU_EMBED      = auto()
     MISTRAL3         = auto()
+    EAGLE3           = auto()
     MISTRAL4         = auto()
     PADDLEOCR        = auto()
     MIMO2            = auto()
@@ -774,14 +771,6 @@ class MODEL_TENSOR(IntEnum):
     V_DS_NORM            = auto() # qwen3vl
     V_DS_FC1             = auto() # qwen3vl
     V_DS_FC2             = auto() # qwen3vl
-    V_MERGER_LN1         = auto() # minicpmv4_6
-    V_MERGER_ATTN_Q      = auto() # minicpmv4_6
-    V_MERGER_ATTN_K      = auto() # minicpmv4_6
-    V_MERGER_ATTN_V      = auto() # minicpmv4_6
-    V_MERGER_ATTN_O      = auto() # minicpmv4_6
-    V_MERGER_DS_LN       = auto() # minicpmv4_6
-    V_MERGER_DS_UP       = auto() # minicpmv4_6
-    V_MERGER_DS_DOWN     = auto() # minicpmv4_6
     V_MM_POST_FC_NORM    = auto() # cogvlm
     V_MM_UP              = auto() # cogvlm
     V_MM_DOWN            = auto() # cogvlm
@@ -859,6 +848,10 @@ class MODEL_TENSOR(IntEnum):
     NEXTN_HNORM          = auto()
     NEXTN_SHARED_HEAD_HEAD = auto()
     NEXTN_SHARED_HEAD_NORM = auto()
+    # EAGLE3 specific tensors
+    EAGLE3_FC          = auto()  # feature fusion layer
+    EAGLE3_HIDDEN_NORM = auto()  # hidden normalization
+    EAGLE3_D2T         = auto()  # draft to target vocabulary mapping
     # lfm2 audio
     A_ENC_NORM_CONV        = auto()
     A_ENC_LINEAR_POS       = auto()
@@ -869,26 +862,6 @@ class MODEL_TENSOR(IntEnum):
     A_ENC_CONV_NORM        = auto() # SSM conv
     A_ENC_CONV_PW1         = auto()
     A_ENC_CONV_PW2         = auto()
-    A_CTC_OUT              = auto()
-    A_CTC_OUT_MID          = auto()
-    A_ENC_ATTN_REL_POS_EMB = auto()
-    # qformer projector
-    A_QF_PROJ_QUERY        = auto()
-    A_QF_PROJ_NORM         = auto()
-    A_QF_PROJ_LINEAR       = auto()
-    A_QF_SELF_ATTN_Q       = auto()
-    A_QF_SELF_ATTN_K       = auto()
-    A_QF_SELF_ATTN_V       = auto()
-    A_QF_SELF_ATTN_O       = auto()
-    A_QF_SELF_ATTN_NORM    = auto()
-    A_QF_CROSS_ATTN_Q      = auto()
-    A_QF_CROSS_ATTN_K      = auto()
-    A_QF_CROSS_ATTN_V      = auto()
-    A_QF_CROSS_ATTN_O      = auto()
-    A_QF_CROSS_ATTN_NORM   = auto()
-    A_QF_FFN_UP            = auto()
-    A_QF_FFN_DOWN          = auto()
-    A_QF_FFN_NORM          = auto()
 
 
 MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
@@ -1011,6 +984,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.RND1:             "rnd1",
     MODEL_ARCH.PANGU_EMBED:      "pangu-embedded",
     MODEL_ARCH.MISTRAL3:         "mistral3",
+    MODEL_ARCH.EAGLE3:           "eagle3",
     MODEL_ARCH.MISTRAL4:         "mistral4",
     MODEL_ARCH.PADDLEOCR:        "paddleocr",
     MODEL_ARCH.MIMO2:            "mimo2",
@@ -1286,14 +1260,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.V_DS_NORM:                 "v.deepstack.{bid}.norm",
     MODEL_TENSOR.V_DS_FC1:                  "v.deepstack.{bid}.fc1",
     MODEL_TENSOR.V_DS_FC2:                  "v.deepstack.{bid}.fc2",
-    MODEL_TENSOR.V_MERGER_LN1:              "v.vit_merger.ln1",
-    MODEL_TENSOR.V_MERGER_ATTN_Q:           "v.vit_merger.attn_q",
-    MODEL_TENSOR.V_MERGER_ATTN_K:           "v.vit_merger.attn_k",
-    MODEL_TENSOR.V_MERGER_ATTN_V:           "v.vit_merger.attn_v",
-    MODEL_TENSOR.V_MERGER_ATTN_O:           "v.vit_merger.attn_out",
-    MODEL_TENSOR.V_MERGER_DS_LN:            "v.vit_merger.ds_ln",
-    MODEL_TENSOR.V_MERGER_DS_UP:            "v.vit_merger.ds_ffn_up",
-    MODEL_TENSOR.V_MERGER_DS_DOWN:          "v.vit_merger.ds_ffn_down",
     MODEL_TENSOR.V_MM_POST_FC_NORM:         "mm.post_fc_norm", # cogvlm
     MODEL_TENSOR.V_MM_UP:                   "mm.up",
     MODEL_TENSOR.V_MM_DOWN:                 "mm.down",
@@ -1376,26 +1342,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.A_ENC_CONV_NORM:           "a.blk.{bid}.conv_norm",
     MODEL_TENSOR.A_ENC_CONV_PW1:            "a.blk.{bid}.conv_pw1",
     MODEL_TENSOR.A_ENC_CONV_PW2:            "a.blk.{bid}.conv_pw2",
-    MODEL_TENSOR.A_CTC_OUT:                 "a.enc_ctc_out",
-    MODEL_TENSOR.A_CTC_OUT_MID:             "a.enc_ctc_out_mid",
-    MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB:    "a.blk.{bid}.attn_rel_pos_emb",
-    # qformer projector
-    MODEL_TENSOR.A_QF_PROJ_QUERY:           "a.proj_query",
-    MODEL_TENSOR.A_QF_PROJ_NORM:            "a.proj_norm",
-    MODEL_TENSOR.A_QF_PROJ_LINEAR:          "a.proj_linear",
-    MODEL_TENSOR.A_QF_SELF_ATTN_Q:          "a.proj_blk.{bid}.self_attn_q",
-    MODEL_TENSOR.A_QF_SELF_ATTN_K:          "a.proj_blk.{bid}.self_attn_k",
-    MODEL_TENSOR.A_QF_SELF_ATTN_V:          "a.proj_blk.{bid}.self_attn_v",
-    MODEL_TENSOR.A_QF_SELF_ATTN_O:          "a.proj_blk.{bid}.self_attn_out",
-    MODEL_TENSOR.A_QF_SELF_ATTN_NORM:       "a.proj_blk.{bid}.self_attn_norm",
-    MODEL_TENSOR.A_QF_CROSS_ATTN_Q:         "a.proj_blk.{bid}.cross_attn_q",
-    MODEL_TENSOR.A_QF_CROSS_ATTN_K:         "a.proj_blk.{bid}.cross_attn_k",
-    MODEL_TENSOR.A_QF_CROSS_ATTN_V:         "a.proj_blk.{bid}.cross_attn_v",
-    MODEL_TENSOR.A_QF_CROSS_ATTN_O:         "a.proj_blk.{bid}.cross_attn_out",
-    MODEL_TENSOR.A_QF_CROSS_ATTN_NORM:      "a.proj_blk.{bid}.cross_attn_norm",
-    MODEL_TENSOR.A_QF_FFN_UP:               "a.proj_blk.{bid}.ffn_up",
-    MODEL_TENSOR.A_QF_FFN_DOWN:             "a.proj_blk.{bid}.ffn_down",
-    MODEL_TENSOR.A_QF_FFN_NORM:             "a.proj_blk.{bid}.ffn_norm",
     # NextN/MTP
     MODEL_TENSOR.NEXTN_EH_PROJ:             "blk.{bid}.nextn.eh_proj",
     MODEL_TENSOR.NEXTN_EMBED_TOKENS:        "blk.{bid}.nextn.embed_tokens",
@@ -1403,6 +1349,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.NEXTN_HNORM:               "blk.{bid}.nextn.hnorm",
     MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD:    "blk.{bid}.nextn.shared_head_head",
     MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM:    "blk.{bid}.nextn.shared_head_norm",
+    MODEL_TENSOR.EAGLE3_FC:                 "fc",
+    MODEL_TENSOR.EAGLE3_HIDDEN_NORM:        "blk.{bid}.hidden_norm",
+    MODEL_TENSOR.EAGLE3_D2T:                "d2t",
 }
 
 MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
@@ -1466,14 +1415,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.V_DS_NORM,
         MODEL_TENSOR.V_DS_FC1,
         MODEL_TENSOR.V_DS_FC2,
-        MODEL_TENSOR.V_MERGER_LN1,
-        MODEL_TENSOR.V_MERGER_ATTN_Q,
-        MODEL_TENSOR.V_MERGER_ATTN_K,
-        MODEL_TENSOR.V_MERGER_ATTN_V,
-        MODEL_TENSOR.V_MERGER_ATTN_O,
-        MODEL_TENSOR.V_MERGER_DS_LN,
-        MODEL_TENSOR.V_MERGER_DS_UP,
-        MODEL_TENSOR.V_MERGER_DS_DOWN,
         MODEL_TENSOR.V_MM_POST_FC_NORM,
         MODEL_TENSOR.V_MM_UP,
         MODEL_TENSOR.V_MM_DOWN,
@@ -1551,26 +1492,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.A_MM_HARD_EMB_NORM,
         MODEL_TENSOR.A_PER_DIM_K_SCALE,
         MODEL_TENSOR.A_PER_DIM_SCALE,
-        MODEL_TENSOR.A_CTC_OUT,
-        MODEL_TENSOR.A_CTC_OUT_MID,
-        MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB,
-        # qformer projector
-        MODEL_TENSOR.A_QF_PROJ_QUERY,
-        MODEL_TENSOR.A_QF_PROJ_NORM,
-        MODEL_TENSOR.A_QF_PROJ_LINEAR,
-        MODEL_TENSOR.A_QF_SELF_ATTN_Q,
-        MODEL_TENSOR.A_QF_SELF_ATTN_K,
-        MODEL_TENSOR.A_QF_SELF_ATTN_V,
-        MODEL_TENSOR.A_QF_SELF_ATTN_O,
-        MODEL_TENSOR.A_QF_SELF_ATTN_NORM,
-        MODEL_TENSOR.A_QF_CROSS_ATTN_Q,
-        MODEL_TENSOR.A_QF_CROSS_ATTN_K,
-        MODEL_TENSOR.A_QF_CROSS_ATTN_V,
-        MODEL_TENSOR.A_QF_CROSS_ATTN_O,
-        MODEL_TENSOR.A_QF_CROSS_ATTN_NORM,
-        MODEL_TENSOR.A_QF_FFN_UP,
-        MODEL_TENSOR.A_QF_FFN_DOWN,
-        MODEL_TENSOR.A_QF_FFN_NORM,
     ],
     MODEL_ARCH.LLAMA: [
         MODEL_TENSOR.TOKEN_EMBD,
@@ -2443,8 +2364,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
         MODEL_TENSOR.FFN_GATE_UP_EXP,
-        MODEL_TENSOR.FFN_GATE_EXP,
-        MODEL_TENSOR.FFN_UP_EXP,
         MODEL_TENSOR.FFN_DOWN_EXP,
         MODEL_TENSOR.ATTN_NORM,
         MODEL_TENSOR.ATTN_POST_NORM,
@@ -3835,6 +3754,24 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_DOWN_EXP,
         MODEL_TENSOR.FFN_UP_EXP,
     ],
+    MODEL_ARCH.EAGLE3: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ROPE_FREQS,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.EAGLE3_FC,
+        MODEL_TENSOR.EAGLE3_HIDDEN_NORM,
+        MODEL_TENSOR.EAGLE3_D2T,
+    ],
     MODEL_ARCH.MISTRAL4: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -3871,7 +3808,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.OUTPUT_NORM,
         MODEL_TENSOR.OUTPUT,
         MODEL_TENSOR.ATTN_NORM,
-        MODEL_TENSOR.ATTN_QKV,
         MODEL_TENSOR.ATTN_Q,
         MODEL_TENSOR.ATTN_K,
         MODEL_TENSOR.ATTN_V,
@@ -3886,10 +3822,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_DOWN_EXP,
         MODEL_TENSOR.FFN_UP_EXP,
         MODEL_TENSOR.FFN_EXP_PROBS_B,
-        MODEL_TENSOR.LAYER_OUT_NORM,
-        MODEL_TENSOR.NEXTN_EH_PROJ,
-        MODEL_TENSOR.NEXTN_ENORM,
-        MODEL_TENSOR.NEXTN_HNORM,
     ],
     MODEL_ARCH.STEP35: [
         MODEL_TENSOR.TOKEN_EMBD,
@@ -4256,8 +4188,6 @@ class VisionProjectorType:
     NEMOTRON_V2_VL = "nemotron_v2_vl"
     HUNYUANOCR     = "hunyuanocr"
     HUNYUANVL      = "hunyuanvl"
-    MINICPMV4_6    = "minicpmv4_6"
-    GRANITE_SPEECH = "granite_speech"  # audio
 
 
 # Items here are (block size, type size)

gguf-py/gguf/gguf_writer.py

@@ -943,9 +943,6 @@ class GGUFWriter:
     def add_attn_output_scale(self, value: float) -> None:
         self.add_float32(Keys.Attention.OUTPUT_SCALE.format(arch=self.arch), value)
 
-    def add_attn_value_scale(self, value: float) -> None:
-        self.add_float32(Keys.Attention.VALUE_SCALE.format(arch=self.arch), value)
-
     def add_attn_temperature_length(self, value: int) -> None:
         self.add_uint32(Keys.Attention.TEMPERATURE_LENGTH.format(arch=self.arch), value)
 
@@ -1263,24 +1260,6 @@ class GGUFWriter:
     def add_audio_stack_factor(self, value: int) -> None:
         self.add_uint32(Keys.ClipAudio.Projector.STACK_FACTOR, value)
 
-    def add_audio_chunk_size(self, value: int) -> None:
-        self.add_uint32(Keys.ClipAudio.CHUNK_SIZE, value)
-
-    def add_audio_conv_kernel_size(self, value: int) -> None:
-        self.add_uint32(Keys.ClipAudio.CONV_KERNEL_SIZE, value)
-
-    def add_audio_max_pos_emb(self, value: int) -> None:
-        self.add_uint32(Keys.ClipAudio.MAX_POS_EMB, value)
-
-    def add_audio_projector_window_size(self, value: int) -> None:
-        self.add_uint32(Keys.ClipAudio.Projector.WINDOW_SIZE, value)
-
-    def add_audio_projector_downsample_rate(self, value: int) -> None:
-        self.add_uint32(Keys.ClipAudio.Projector.DOWNSAMPLE_RATE, value)
-
-    def add_audio_projector_head_count(self, value: int) -> None:
-        self.add_uint32(Keys.ClipAudio.Projector.HEAD_COUNT, value)
-
     def add_xielu_alpha_p(self, values: Sequence[float]):
         self.add_array(Keys.xIELU.ALPHA_P, values)
 

gguf-py/gguf/tensor_mapping.py

@@ -18,6 +18,7 @@ class TensorNameMap:
             "tok_embeddings",                            # llama-pth
             "embeddings.word_embeddings",                # bert nomic-bert
             "embeddings.tok_embeddings",                 # modern-bert
+            "language_model.embedding.word_embeddings",  # persimmon
             "wte",                                       # gpt2
             "transformer.embd.wte",                      # phi2
             "model.tok_embeddings",                      # internlm2
@@ -31,6 +32,7 @@ class TensorNameMap:
             "rwkv.embeddings",                           # rwkv6
             "model.embeddings",                          # rwkv7
             "model.word_embeddings",                     # bailingmoe
+            "language_model.model.embed_tokens",         # llama4
             "encoder",                                   # neobert
             "model.transformer.wte",                     # llada
             "embed_tokens",                              # qwen3-embedding
@@ -92,6 +94,7 @@ class TensorNameMap:
             "norm",                                    # llama-pth
             "transformer.norm_f",                      # mpt dbrx
             "ln_f",                                    # refact bloom qwen gpt2
+            "language_model.encoder.final_layernorm",  # persimmon
             "model.final_layernorm",                   # persimmon
             "lm_head.ln",                              # phi2
             "model.norm_f",                            # mamba-qbert
@@ -155,21 +158,6 @@ class TensorNameMap:
         MODEL_TENSOR.V_ENC_MSFA_NORM: (
             "model.vision_tower.timm_model.msfa.norm", # gemma3n
         ),
-        MODEL_TENSOR.A_CTC_OUT: (
-            "encoder.out",
-        ),
-        MODEL_TENSOR.A_CTC_OUT_MID: (
-            "encoder.out_mid",
-        ),
-        MODEL_TENSOR.A_QF_PROJ_QUERY: (
-            "projector.query",
-        ),
-        MODEL_TENSOR.A_QF_PROJ_NORM: (
-            "projector.qformer.layernorm",
-        ),
-        MODEL_TENSOR.A_QF_PROJ_LINEAR: (
-            "projector.linear",
-        ),
     }
 
     block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {
@@ -183,6 +171,7 @@ class TensorNameMap:
             "transformer.h.{bid}.ln_mlp",                           # falcon40b
             "model.layers.{bid}.input_layernorm",                   # llama-hf nemotron olmoe phimoe granite-hybrid
             "layers.{bid}.attention_norm",                          # llama-pth
+            "language_model.encoder.layers.{bid}.input_layernorm",  # persimmon
             "model.layers.{bid}.ln1",                               # yi
             "h.{bid}.ln_1",                                         # gpt2
             "transformer.h.{bid}.ln",                               # phi2
@@ -226,6 +215,7 @@ class TensorNameMap:
             "transformer.blocks.{bid}.norm_attn_norm.attn.Wqkv",                   # dbrx
             "transformer.h.{bid}.self_attention.query_key_value",                  # falcon
             "h.{bid}.self_attention.query_key_value",                              # bloom
+            "language_model.encoder.layers.{bid}.self_attention.query_key_value",  # persimmon
             "model.layers.{bid}.self_attn.query_key_value",                        # persimmon
             "model.layers.{bid}.attention.query_key_value",                        # bailingmoe2
             "h.{bid}.attn.c_attn",                                                 # gpt2
@@ -316,6 +306,7 @@ class TensorNameMap:
             "layers.{bid}.attn.Wo",                                         # modern-bert
             "transformer.layer.{bid}.attention.out_lin",                    # distillbert
             "transformer.h.{bid}.attn.out_proj",                            # gpt-j
+            "language_model.encoder.layers.{bid}.self_attention.dense",     # persimmon
             "model.layers.{bid}.self_attn.dense",                           # persimmon
             "model.layers.{bid}.attention.dense",                           # bailingmoe2
             "h.{bid}.attn.c_proj",                                          # gpt2
@@ -382,6 +373,7 @@ class TensorNameMap:
             "transformer.blocks.{bid}.norm_2",                               # mpt
             "model.layers.{bid}.post_attention_layernorm",                   # llama-hf nemotron olmoe phimoe
             "layers.{bid}.ffn_norm",                                         # llama-pth
+            "language_model.encoder.layers.{bid}.post_attention_layernorm",  # persimmon
             "model.layers.{bid}.ln2",                                        # yi
             "h.{bid}.ln_2",                                                  # gpt2
             "model.layers.{bid}.ffn_norm",                                   # internlm2
@@ -483,6 +475,7 @@ class TensorNameMap:
             "transformer.layer.{bid}.ffn.lin1",                       # distillbert
             "transformer.h.{bid}.mlp.fc_in",                          # gpt-j
             "transformer.h.{bid}.mlp.linear_3",                       # refact
+            "language_model.encoder.layers.{bid}.mlp.dense_h_to_4h",  # persimmon
             "model.layers.{bid}.mlp.dense_h_to_4h",                   # persimmon
             "transformer.h.{bid}.mlp.w1",                             # qwen
             "h.{bid}.mlp.c_fc",                                       # gpt2
@@ -615,6 +608,7 @@ class TensorNameMap:
             "layers.{bid}.mlp.Wo",                                    # modern-bert
             "transformer.layer.{bid}.ffn.lin2",                       # distillbert
             "transformer.h.{bid}.mlp.fc_out",                         # gpt-j
+            "language_model.encoder.layers.{bid}.mlp.dense_4h_to_h",  # persimmon
             "model.layers.{bid}.mlp.dense_4h_to_h",                   # persimmon
             "h.{bid}.mlp.c_proj",                                     # gpt2
             "transformer.h.{bid}.mlp.fc2",                            # phi2
@@ -669,7 +663,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.ATTN_Q_NORM: (
-            "encoder.layers.{bid}.self_attention.q_layernorm",
+            "language_model.encoder.layers.{bid}.self_attention.q_layernorm",
             "model.layers.{bid}.self_attn.q_layernorm",                       # persimmon
             "model.layers.{bid}.self_attn.query_layernorm",                   # hunyuan
             "model.layers.{bid}.attention.query_layernorm",                   # bailingmoe2
@@ -685,7 +679,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.ATTN_K_NORM: (
-            "encoder.layers.{bid}.self_attention.k_layernorm",
+            "language_model.encoder.layers.{bid}.self_attention.k_layernorm",
             "model.layers.{bid}.self_attn.k_layernorm",                       # persimmon
             "model.layers.{bid}.self_attn.key_layernorm",                     # hunyuan
             "model.layers.{bid}.attention.key_layernorm",                     # bailingmoe2
@@ -701,7 +695,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.ROPE_FREQS: (
-            "encoder.layers.{bid}.self_attention.rotary_emb.inv_freq",  # persimmon
+            "language_model.encoder.layers.{bid}.self_attention.rotary_emb.inv_freq",  # persimmon
         ),
 
         MODEL_TENSOR.LAYER_OUT_NORM: (
@@ -1399,7 +1393,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_EMBD_PATCH: (
             "vision_tower.vision_model.embeddings.patch_embedding",
-            "model.vision_tower.embeddings.patch_embedding", # minicpmv4_6
             "model.vision_tower.embeddings.patch_embeddings.projection", # Intern-S1
             "vpm.embeddings.patch_embedding",
             "model.vision_model.embeddings.patch_embedding", # SmolVLM
@@ -1425,7 +1418,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_EMBD_POS: (
             "vision_tower.vision_model.embeddings.position_embedding",
-            "model.vision_tower.embeddings.position_embedding", # minicpmv4_6
             "model.vision_tower.embeddings.position_embeddings", # Intern-S1
             "vpm.embeddings.position_embedding",
             "model.vision_model.embeddings.position_embedding", # SmolVLM
@@ -1462,7 +1454,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_ATTN_Q: (
             "vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj",
-            "model.vision_tower.encoder.layers.{bid}.self_attn.q_proj", # minicpmv4_6
             "model.vision_tower.encoder.layer.{bid}.attention.q_proj", # Intern-S1
             "vpm.encoder.layers.{bid}.self_attn.q_proj",
             "model.vision_model.encoder.layers.{bid}.self_attn.q_proj", # SmolVLM
@@ -1486,7 +1477,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_ATTN_K: (
             "vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj",
-            "model.vision_tower.encoder.layers.{bid}.self_attn.k_proj", # minicpmv4_6
             "model.vision_tower.encoder.layer.{bid}.attention.k_proj", # Intern-S1
             "vpm.encoder.layers.{bid}.self_attn.k_proj",
             "model.vision_model.encoder.layers.{bid}.self_attn.k_proj", # SmolVLM
@@ -1510,7 +1500,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_ATTN_V: (
             "vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj",
-            "model.vision_tower.encoder.layers.{bid}.self_attn.v_proj", # minicpmv4_6
             "model.vision_tower.encoder.layer.{bid}.attention.v_proj", # Intern-S1
             "vpm.encoder.layers.{bid}.self_attn.v_proj",
             "model.vision_model.encoder.layers.{bid}.self_attn.v_proj", # SmolVLM
@@ -1527,7 +1516,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_INPUT_NORM: (
             "vision_tower.vision_model.encoder.layers.{bid}.layer_norm1",
-            "model.vision_tower.encoder.layers.{bid}.layer_norm1", # minicpmv4_6
             "vision_tower.vision_model.encoder.layers.{bid}.norm1", # InternVL
             "model.vision_tower.encoder.layer.{bid}.layernorm_before", # Intern-S1
             "vpm.encoder.layers.{bid}.layer_norm1",
@@ -1548,7 +1536,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_ATTN_O: (
             "vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj",
-            "model.vision_tower.encoder.layers.{bid}.self_attn.out_proj", # minicpmv4_6
             "vision_tower.vision_model.encoder.layers.{bid}.attn.proj", # InternVL
             "model.vision_tower.encoder.layer.{bid}.attention.projection_layer", # Intern-S1
             "vpm.encoder.layers.{bid}.self_attn.out_proj",
@@ -1571,7 +1558,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
             "vision_tower.vision_model.encoder.layers.{bid}.layer_norm2",
-            "model.vision_tower.encoder.layers.{bid}.layer_norm2", # minicpmv4_6
             "vision_tower.vision_model.encoder.layers.{bid}.norm2", # InternVL
             "model.vision_tower.encoder.layer.{bid}.layernorm_after", # Intern-S1
             "vpm.encoder.layers.{bid}.layer_norm2",
@@ -1593,7 +1579,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_FFN_UP: (
             "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",
-            "model.vision_tower.encoder.layers.{bid}.mlp.fc1", # minicpmv4_6
             "model.vision_tower.encoder.layer.{bid}.mlp.fc1", # Intern-S1
             "vpm.encoder.layers.{bid}.mlp.fc1",
             "model.vision_model.encoder.layers.{bid}.mlp.fc1", # SmolVLM, gemma3
@@ -1622,7 +1607,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_ENC_FFN_DOWN: (
             "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",
-            "model.vision_tower.encoder.layers.{bid}.mlp.fc2", # minicpmv4_6
             "model.vision_tower.encoder.layer.{bid}.mlp.fc2", # Intern-S1
             "vpm.encoder.layers.{bid}.mlp.fc2",
             "model.vision_model.encoder.layers.{bid}.mlp.fc2", # SmolVLM, gemma3
@@ -1678,7 +1662,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.V_POST_NORM: (
             "vision_tower.vision_model.post_layernorm",
-            "model.vision_tower.post_layernorm", # minicpmv4_6
             "model.vision_model.post_layernorm", # SmolVLM
             "vision_model.layernorm_post", # llama4
             "visual.merger.ln_q", # qwen2vl
@@ -1707,7 +1690,6 @@ class TensorNameMap:
             "mlp_AR.pre_norm", # PaddleOCR-VL
             "merger.ln_q",
             "vision_tower.merger.ln_q", # dots.ocr
-            "model.merger.mlp.0.pre_norm", # minicpmv4_6
         ),
 
         MODEL_TENSOR.V_MM_SOFT_EMB_NORM: (
@@ -1781,38 +1763,6 @@ class TensorNameMap:
             "model.visual.deepstack_merger_list.{bid}.linear_fc2", # deepstack in qwen3vl
         ),
 
-        MODEL_TENSOR.V_MERGER_LN1: (
-            "model.vision_tower.vit_merger.layer_norm1", # minicpmv4_6
-        ),
-
-        MODEL_TENSOR.V_MERGER_ATTN_Q: (
-            "model.vision_tower.vit_merger.self_attn.q_proj", # minicpmv4_6
-        ),
-
-        MODEL_TENSOR.V_MERGER_ATTN_K: (
-            "model.vision_tower.vit_merger.self_attn.k_proj", # minicpmv4_6
-        ),
-
-        MODEL_TENSOR.V_MERGER_ATTN_V: (
-            "model.vision_tower.vit_merger.self_attn.v_proj", # minicpmv4_6
-        ),
-
-        MODEL_TENSOR.V_MERGER_ATTN_O: (
-            "model.vision_tower.vit_merger.self_attn.out_proj", # minicpmv4_6
-        ),
-
-        MODEL_TENSOR.V_MERGER_DS_LN: (
-            "model.vision_tower.vit_merger.pre_norm", # minicpmv4_6
-        ),
-
-        MODEL_TENSOR.V_MERGER_DS_UP: (
-            "model.vision_tower.vit_merger.linear_1", # minicpmv4_6
-        ),
-
-        MODEL_TENSOR.V_MERGER_DS_DOWN: (
-            "model.vision_tower.vit_merger.linear_2", # minicpmv4_6
-        ),
-
         MODEL_TENSOR.V_SAM_POS_EMBD: (
             "model.sam_model.pos_embed",
         ),
@@ -1872,13 +1822,11 @@ class TensorNameMap:
         MODEL_TENSOR.V_MM_UP: (
             "model.vision.linear_proj.dense_h_to_4h", # cogvlm
             "visual.merger.up_proj", # glm4v
-            "model.merger.mlp.0.linear_1", # minicpmv4_6
         ),
 
         MODEL_TENSOR.V_MM_DOWN: (
             "model.vision.linear_proj.dense_4h_to_h", # cogvlm
             "visual.merger.down_proj", # glm4v
-            "model.merger.mlp.0.linear_2", # minicpmv4_6
         ),
 
         MODEL_TENSOR.V_MM_GATE: (
@@ -1942,7 +1890,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_INP_PROJ: (
             "conformer.subsample_conv_projection.input_proj_linear", # gemma4
-            "encoder.input_linear",
         ),
 
         MODEL_TENSOR.A_ENC_CONV2D: (
@@ -1965,7 +1912,6 @@ class TensorNameMap:
             "conformer.layers.{bid}.self_attn.linear_q", # lfm2
             "conformer.layers.{bid}.attention.attn.q_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.q_proj", # gemma4
-            "encoder.layers.{bid}.attn.to_q", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_ATTN_K: (
@@ -1973,7 +1919,6 @@ class TensorNameMap:
             "conformer.layers.{bid}.self_attn.linear_k", # lfm2
             "conformer.layers.{bid}.attention.attn.k_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.k_proj", # gemma4
-            "encoder.layers.{bid}.attn.to_k", # granite_speech (split from to_kv)
         ),
 
         MODEL_TENSOR.A_ENC_ATTN_V: (
@@ -1981,7 +1926,6 @@ class TensorNameMap:
             "conformer.layers.{bid}.self_attn.linear_v", # lfm2
             "conformer.layers.{bid}.attention.attn.v_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.v_proj", # gemma4
-            "encoder.layers.{bid}.attn.to_v", # granite_speech (split from to_kv)
         ),
 
         MODEL_TENSOR.A_ENC_ATTN_K_REL: (
@@ -2009,7 +1953,6 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox
             "conformer.layers.{bid}.norm_self_att", # lfm2
             "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
-            "encoder.layers.{bid}.attn.pre_norm", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_OUTPUT: (
@@ -2017,21 +1960,18 @@ class TensorNameMap:
             "conformer.layers.{bid}.self_attn.linear_out", # lfm2
             "conformer.layers.{bid}.attention.post", # gemma3n
             "conformer.layers.{bid}.self_attn.post", # gemma4
-            "encoder.layers.{bid}.attn.to_out", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_OUTPUT_NORM: (
             "audio_tower.layers.{bid}.final_layer_norm", # ultravox
             "conformer.layers.{bid}.norm_out", # lfm2
             "conformer.layers.{bid}.attention.post_norm", # gemma3n
-            "encoder.layers.{bid}.post_norm", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_FFN_NORM: (
             "conformer.layers.{bid}.norm_feed_forward1", # lfm2
             "conformer.layers.{bid}.ffw_layer_start.pre_layer_norm", # gemma3n
             "conformer.layers.{bid}.feed_forward1.pre_layer_norm", # gemma4
-            "encoder.layers.{bid}.ff1.pre_norm", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_FFN_POST_NORM: (
@@ -2048,7 +1988,6 @@ class TensorNameMap:
             "conformer.layers.{bid}.feed_forward1.linear1", # lfm2
             "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
             "conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4
-            "encoder.layers.{bid}.ff1.up_proj", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_FFN_GATE: (),
@@ -2058,28 +1997,24 @@ class TensorNameMap:
             "conformer.layers.{bid}.feed_forward1.linear2", # lfm2
             "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
             "conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4
-            "encoder.layers.{bid}.ff1.down_proj", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_FFN_UP_1: (
             "conformer.layers.{bid}.feed_forward2.linear1", # lfm2
             "conformer.layers.{bid}.ffw_layer_end.ffw_layer_1", # gemma3n
             "conformer.layers.{bid}.feed_forward2.ffw_layer_1", # gemma4
-            "encoder.layers.{bid}.ff2.up_proj", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_FFN_DOWN_1: (
             "conformer.layers.{bid}.feed_forward2.linear2", # lfm2
             "conformer.layers.{bid}.ffw_layer_end.ffw_layer_2", # gemma3n
             "conformer.layers.{bid}.feed_forward2.ffw_layer_2", # gemma4
-            "encoder.layers.{bid}.ff2.down_proj", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_FFN_NORM_1: (
             "conformer.layers.{bid}.norm_feed_forward2", # lfm2
             "conformer.layers.{bid}.ffw_layer_end.pre_layer_norm", # gemma3n
             "conformer.layers.{bid}.feed_forward2.pre_layer_norm", # gemma4
-            "encoder.layers.{bid}.ff2.pre_norm", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: (
@@ -2136,31 +2071,26 @@ class TensorNameMap:
         MODEL_TENSOR.A_ENC_CONV_DW: (
             "conformer.layers.{bid}.conv.depthwise_conv", # lfm2
             "conformer.layers.{bid}.lconv1d.depthwise_conv1d", # gemma3n
-            "encoder.layers.{bid}.conv.depth_conv.conv", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_CONV_NORM: (
             "conformer.layers.{bid}.conv.batch_norm", # lfm2
             "conformer.layers.{bid}.lconv1d.pre_layer_norm", # gemma3n
-            "encoder.layers.{bid}.conv.batch_norm", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_CONV_PW1: (
             "conformer.layers.{bid}.conv.pointwise_conv1", # lfm2
             "conformer.layers.{bid}.lconv1d.linear_start", # gemma3n
-            "encoder.layers.{bid}.conv.up_conv", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_CONV_PW2: (
             "conformer.layers.{bid}.conv.pointwise_conv2", # lfm2
             "conformer.layers.{bid}.lconv1d.linear_end", # gemma3n
-            "encoder.layers.{bid}.conv.down_conv", # granite_speech
         ),
 
         MODEL_TENSOR.A_ENC_NORM_CONV: (
             "conformer.layers.{bid}.norm_conv", # lfm2
             "conformer.layers.{bid}.lconv1d.conv_norm", # gemma3n
-            "encoder.layers.{bid}.conv.norm", # granite_speech
         ),
 
         MODEL_TENSOR.A_PER_DIM_K_SCALE: (
@@ -2184,62 +2114,6 @@ class TensorNameMap:
             "model.embed_audio.soft_embedding_norm", # gemma3n
         ),
 
-        MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB: (
-            "encoder.layers.{bid}.attn.rel_pos_emb.weight",
-        ),
-
-        MODEL_TENSOR.A_QF_SELF_ATTN_Q: (
-            "projector.qformer.encoder.layer.{bid}.attention.attention.query",
-        ),
-
-        MODEL_TENSOR.A_QF_SELF_ATTN_K: (
-            "projector.qformer.encoder.layer.{bid}.attention.attention.key",
-        ),
-
-        MODEL_TENSOR.A_QF_SELF_ATTN_V: (
-            "projector.qformer.encoder.layer.{bid}.attention.attention.value",
-        ),
-
-        MODEL_TENSOR.A_QF_SELF_ATTN_O: (
-            "projector.qformer.encoder.layer.{bid}.attention.output.dense",
-        ),
-
-        MODEL_TENSOR.A_QF_SELF_ATTN_NORM: (
-            "projector.qformer.encoder.layer.{bid}.attention.output.LayerNorm",
-        ),
-
-        MODEL_TENSOR.A_QF_CROSS_ATTN_Q: (
-            "projector.qformer.encoder.layer.{bid}.crossattention.attention.query",
-        ),
-
-        MODEL_TENSOR.A_QF_CROSS_ATTN_K: (
-            "projector.qformer.encoder.layer.{bid}.crossattention.attention.key",
-        ),
-
-        MODEL_TENSOR.A_QF_CROSS_ATTN_V: (
-            "projector.qformer.encoder.layer.{bid}.crossattention.attention.value",
-        ),
-
-        MODEL_TENSOR.A_QF_CROSS_ATTN_O: (
-            "projector.qformer.encoder.layer.{bid}.crossattention.output.dense",
-        ),
-
-        MODEL_TENSOR.A_QF_CROSS_ATTN_NORM: (
-            "projector.qformer.encoder.layer.{bid}.crossattention.output.LayerNorm",
-        ),
-
-        MODEL_TENSOR.A_QF_FFN_UP: (
-            "projector.qformer.encoder.layer.{bid}.intermediate_query.dense",
-        ),
-
-        MODEL_TENSOR.A_QF_FFN_DOWN: (
-            "projector.qformer.encoder.layer.{bid}.output_query.dense",
-        ),
-
-        MODEL_TENSOR.A_QF_FFN_NORM: (
-            "projector.qformer.encoder.layer.{bid}.output_query.LayerNorm",
-        ),
-
         # NextN/MTP tensors
         MODEL_TENSOR.NEXTN_EH_PROJ: (
             "model.layers.{bid}.eh_proj",