fix split

- add internal all reduce
- add SET_TENSOR_2D/GET_TENSOR_2D for strided transfers
- add LRU graph cache

.github/workflows/build-apple.yml

@@ -60,6 +60,7 @@ jobs:
             -DCMAKE_BUILD_RPATH="@loader_path" \
             -DLLAMA_FATAL_WARNINGS=ON \
             -DLLAMA_BUILD_BORINGSSL=ON \
+            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=OFF \
             -DGGML_METAL_SHADER_DEBUG=ON \
             -DGGML_RPC=ON \
@@ -126,6 +127,7 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
+            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_OPENSSL=OFF \
             -DLLAMA_BUILD_APP=OFF \
@@ -176,6 +178,7 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
+            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_BUILD_COMMON=OFF \
             -DLLAMA_BUILD_APP=OFF \
@@ -209,6 +212,7 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
+            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_BUILD_COMMON=OFF \
             -DLLAMA_BUILD_APP=OFF \
@@ -253,6 +257,7 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
+            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_OPENSSL=OFF \
             -DLLAMA_BUILD_APP=OFF \

.github/workflows/build-self-hosted.yml

@@ -71,26 +71,6 @@ jobs:
           nvidia-smi
           GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
 
-  gpu-rocm:
-    runs-on: [self-hosted, Linux, AMD]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Test
-        id: ggml-ci
-        # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
-        # issue on integrated RDNA3.5 (gfx1151) where batched inference returns
-        # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
-        # restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
-        env:
-          HIP_LAUNCH_BLOCKING: "1"
-        run: |
-          rocminfo
-          GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
   gpu-vulkan-nvidia-cm:
     runs-on: [self-hosted, Linux, NVIDIA]
 

.github/workflows/release.yml

@@ -93,13 +93,13 @@ jobs:
           - build: 'arm64'
             arch: 'arm64'
             os: macos-26
-            defines: "-DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"
+            defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"
           # TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23780)
           #       in order to enable it again, we have to provision dedicated runners  to run it
           #- build: 'arm64-kleidiai'
           #  arch: 'arm64'
           #  os: macos-14
-          #  defines: "-DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 -DGGML_CPU_KLEIDIAI=ON"
+          #  defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 -DGGML_CPU_KLEIDIAI=ON"
           - build: 'x64'
             arch: 'x64'
             os: macos-15-intel
@@ -1402,6 +1402,7 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
+            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_OPENSSL=OFF \
             -DLLAMA_BUILD_APP=OFF \

SECURITY.md

@@ -21,18 +21,11 @@ Please disclose it as a private [security advisory](https://github.com/ggml-org/
 
 A team of volunteers on a reasonable-effort basis maintains this project. As such, please give us at least 90 days to work on a fix before public exposure.
 
-### AI-powered code scan
-
-llama.cpp has an AI security scanner that scans the code periodically. The full prompts and tool set can be found in [ggml-org/security-scan-prompt](https://github.com/ggml-org/security-scan-prompt).
-
-We greatly appreciate reports that reflect genuine research effort, and we are happy to spend our time reviewing them. Findings that an autonomous AI agent can surface on its own add little on top of the scans we already run.
-
 ### Requirements
 
 Before submitting your report, ensure you meet the following requirements:
 
 - You have read this policy and fully understand it.
-- You have searched for existing discussions of the issue. If it has already been reported, your report will likely be rejected as a duplicate.
 - AI is only permitted in an assistive capacity as stated in [AGENTS.md](AGENTS.md). We do not accept reports that are written exclusively by AI.
 - Your report must include a working Proof-of-Concept in the form of a script and/or attached files.
 
@@ -53,8 +46,6 @@ Only vulnerabilities that fall within these parts of the project are considered
 
 Note that none of the topics under [Using llama.cpp securely](#using-llamacpp-securely) are considered vulnerabilities in LLaMA C++.
 
-Denial-of-Service (DoS) bugs are generally not treated as vulnerabilities. We don't reject them outright, but we look at them case-by-case and only accept those that are genuinely worth fixing.
-
 For vulnerabilities that fall within the `vendor` directory, please report them directly to the third-party project.
 
 ## Using llama.cpp securely

build-xcframework.sh

@@ -17,6 +17,7 @@ LLAMA_BUILD_MTMD=ON
 GGML_METAL=ON
 GGML_METAL_EMBED_LIBRARY=ON
 GGML_BLAS_DEFAULT=ON
+GGML_METAL_USE_BF16=ON
 GGML_OPENMP=OFF
 
 COMMON_C_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
@@ -43,6 +44,7 @@ COMMON_CMAKE_ARGS=(
     -DGGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY}
     -DGGML_BLAS_DEFAULT=${GGML_BLAS_DEFAULT}
     -DGGML_METAL=${GGML_METAL}
+    -DGGML_METAL_USE_BF16=${GGML_METAL_USE_BF16}
     -DGGML_NATIVE=OFF
     -DGGML_OPENMP=${GGML_OPENMP}
 )

ci/run.sh

@@ -10,9 +10,6 @@
 # # with CUDA support
 # GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
 #
-# # with ROCm support
-# GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ./tmp/results ./tmp/mnt
-#
 # # with SYCL support
 # GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
 #
@@ -92,7 +89,7 @@ if [ ! -z ${GG_BUILD_CUDA} ]; then
 fi
 
 if [ ! -z ${GG_BUILD_ROCM} ]; then
-    CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_HIP_COMPILER=$(hipconfig -l)/clang -DGGML_HIP=ON -DGGML_HIP_ROCWMMA_FATTN=ON"
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_HIP=ON"
     if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then
         echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)"
         exit 1
@@ -643,52 +640,39 @@ function gg_sum_rerank_tiny {
 
 function gg_check_build_requirements {
     if ! command -v git &> /dev/null; then
-        gg_printf 'git not found, please install\n'
-        exit 1
+        gg_printf 'git not found, please install'
     fi
 
     if ! command -v git-lfs &> /dev/null; then
-        gg_printf 'git-lfs not found, please install\n'
-        exit 1
-    fi
-
-    if ! git config --get filter.lfs.clean &> /dev/null; then
-        gg_printf 'git-lfs not initialized, please run `git lfs install`\n'
-        exit 1
+        gg_printf 'git-lfs not found, please install'
     fi
 
     if ! command -v wget &> /dev/null; then
-        gg_printf 'wget not found, please install\n'
-        exit 1
+        gg_printf 'wget not found, please install'
     fi
 
     if ! command -v python3 &> /dev/null; then
-        gg_printf 'python3 not found, please install\n'
-        exit 1
+        gg_printf 'python3 not found, please install'
     fi
 
     if ! command -v pip3 &> /dev/null; then
-        gg_printf 'pip3 not found, please install\n'
-        exit 1
+        gg_printf 'pip3 not found, please install'
     fi
 
     if ! python3 -m ensurepip --help &> /dev/null; then
-        gg_printf 'ensurepip not found, please install python3-venv package\n'
-        exit 1
+        gg_printf 'ensurepip not found, please install python3-venv package'
     fi
 
     if ! command -v cmake &> /dev/null; then
-        gg_printf 'cmake not found, please install\n'
-        exit 1
+        gg_printf 'cmake not found, please install'
     fi
 
     if ! command -v ccache &> /dev/null; then
-        gg_printf 'ccache not found, please consider installing for faster builds\n'
+        gg_printf 'ccache not found, please consider installing for faster builds'
     fi
 
     if ! command -v ctest &> /dev/null; then
-        gg_printf 'ctest not found, please install\n'
-        exit 1
+        gg_printf 'ctest not found, please install'
     fi
 }
 

common/fit.cpp

@@ -136,10 +136,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
         devs.push_back(llama_model_get_device(model, i));
     }
 
-    hp_ngl         = llama_model_n_layer(model);
-    if (mparams->load_mtp) {
-        hp_ngl    += llama_model_n_layer_nextn(model);
-    }
+    hp_ngl         = llama_model_n_layer(model) + llama_model_n_layer_nextn(model);
     hp_n_ctx_train = llama_model_n_ctx_train(model);
     hp_n_expert    = llama_model_n_expert(model);
 

conversion/__init__.py

@@ -70,7 +70,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "Exaone4ForCausalLM": "exaone",
     "ExaoneForCausalLM": "exaone",
     "ExaoneMoEForCausalLM": "exaone",
-    "ExaoneMoeForCausalLM": "exaone",
     "FalconForCausalLM": "falcon",
     "FalconH1ForCausalLM": "falcon_h1",
     "FalconMambaForCausalLM": "mamba",

conversion/deepseek.py

@@ -17,11 +17,8 @@ from .base import LazyTorchTensor, MmprojModel, ModelBase, TextModel, gguf, logg
 from .qwen import QwenModel
 
 
-@ModelBase.register("DeepseekOCRForCausalLM")
+@ModelBase.register("DeepseekOCRForCausalLM", "UnlimitedOCRForCausalLM")
 class DeepseekOCRVisionModel(MmprojModel):
-    # HF dynamic_preprocess() max_num, which differs per model
-    preproc_max_tiles = 9
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR
@@ -46,9 +43,6 @@ class DeepseekOCRVisionModel(MmprojModel):
         # @bluebread: there's no window_size in config but just add it here anyway
         self.gguf_writer.add_vision_window_size(self.hparams.get("window_size", 14))
 
-        self.gguf_writer.add_vision_preproc_min_tiles(2)
-        self.gguf_writer.add_vision_preproc_max_tiles(self.preproc_max_tiles)
-
         # SAM configuration
         sam_hparams = hparams['sam']
         self.gguf_writer.add_vision_sam_layers_count(sam_hparams['layers'])
@@ -99,15 +93,8 @@ class DeepseekOCRVisionModel(MmprojModel):
         return super().filter_tensors((name, gen))
 
 
-@ModelBase.register("UnlimitedOCRForCausalLM")
-class UnlimitedOCRVisionModel(DeepseekOCRVisionModel):
-    preproc_max_tiles = 32
-
-
 @ModelBase.register("DeepseekOCR2ForCausalLM")
 class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
-    preproc_max_tiles = 6
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR2
@@ -533,13 +520,6 @@ class DeepseekV4Model(TextModel):
         for key, value in raw_hparams.items():
             self.hparams.setdefault(key, value)
 
-        # workaround for special rope_parameters (main/compress) in transformers 5.x
-        if self.rope_parameters.get("full_attention", self.rope_parameters).get("rope_type") is None:
-            if (rope_scaling := raw_hparams.get("rope_scaling")) is not None:
-                if "rope_type" not in rope_scaling and (rope_type := rope_scaling.get("type")) is not None:
-                    rope_scaling["rope_type"] = rope_type
-                self.rope_parameters.update(**rope_scaling)
-
         self.block_count = self.hparams["num_hidden_layers"]
         if self.mtp_only:
             self.block_count += self.hparams.get("num_nextn_predict_layers", 0)

conversion/exaone.py

@@ -123,9 +123,7 @@ class Exaone4Model(TextModel):
                 yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))
 
 
-# note: transformers >= 5.1 renamed the class to "ExaoneMoeForCausalLM" (lowercase 'e'),
-#       so accept both spellings - LG AI have updated the configs of already-released models
-@ModelBase.register("ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM")
+@ModelBase.register("ExaoneMoEForCausalLM")
 class ExaoneMoEModel(Exaone4Model):
     model_arch = gguf.MODEL_ARCH.EXAONE_MOE
 

docs/backend/SYCL.md

@@ -449,8 +449,6 @@ Or
 use 1 SYCL GPUs: [0] with Max compute units:512
 ```
 
-User can use the device management in [docs/multi-gpu.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md), like parameter `--device SYCL0,SYCL1` to assign one or more devices.
-
 ## Windows
 
 ### Install GPU driver
@@ -765,7 +763,6 @@ Or
 use 1 SYCL GPUs: [0] with Max compute units:512
 ```
 
-User can use the device management in [docs/multi-gpu.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md), like parameter `--device SYCL0,SYCL1` to assign one or more devices.
 
 ## Environment Variable
 
@@ -898,45 +895,6 @@ Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spo
     set UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1
   ```
 
-- When I set `SYCL_CACHE_PERSISTENT=1` in running time, I meet crash.
-
-  `SYCL_CACHE_PERSISTENT=1` is not recommended by llama.cpp SYCL backend.
-  When cache is enabled, SYCL runtime will try to cache and reuse JIT-compiled binaries.
-
-  We find some AI will tell user this cmd to speed up SYCL backend. It only speeds up the startup to skip the JIT process, instead of running speed.
-
-  It will bring negative impact when the SYCL binary file is changed frequently in your running environment. The new & old codes mix will lead to crash.
-
-  Compare to the benefit, it has brought more failed cases.
-  If you are not familiar with the SYCL compiler principle of JIT and AOT, please don't use it.
-
-  To restore, you need to remove the local cache: `~/.cache/libsycl_cache/` and execute `unset SYCL_CACHE_PERSISTENT` in running time.
-
-- How to use iGPU and dGPU in same time?
-
-  1. Detect the devices in your running time.
-  ```
-  source /opt/intel/oneapi/setvars.sh
-  ./build/bin/llama-server --list-devices
-
-  or
-  ./build/bin/llama-cli --list-devices
-  ./build/bin/llama-bench --list-devices
-  ./build/bin/llama-completion --list-devices
-
-  Available devices:
-    SYCL0: Intel(R) Arc(TM) A770 Graphics (15473 MiB, 15473 MiB free)
-    SYCL1: Intel(R) UHD Graphics 770 (59675 MiB, 44986 MiB free)
-  ```
-
-  The dGPU will be in the head of this list and iGPU will be the end.
-  If not all GPUs are listed, please check the env var: ONEAPI_DEVICE_SELECTOR and unset it.
-
-  2. Set the iGPU and dGPU
-
-  Set the iGPU and dGPU by `./build/bin/llama-server --device SYCL0,SYCL1,SYCLxxx`.
-
-
 ### **GitHub contribution**:
 Please add the `[SYCL]` prefix/tag in issues/PRs titles to help the SYCL contributors to check/address them without delay.
 

docs/ops.md

@@ -15,7 +15,7 @@ Legend:
 | Operation | BLAS | CANN | CPU | CUDA | ET | MTL | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
 |-----------|------|------|------|------|------|------|------|------|------|------|------|------|
 |                              ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                              ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
 |                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             ADD1 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
@@ -41,9 +41,9 @@ Legend:
 |                             DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                              DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                     DSV4_HC_COMB | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                     DSV4_HC_POST | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                      DSV4_HC_PRE | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                     DSV4_HC_COMB | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                     DSV4_HC_POST | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                      DSV4_HC_PRE | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |                              DUP | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                              ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                              EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
@@ -59,7 +59,7 @@ Legend:
 |                             GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                       GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
+|                         GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                    GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ |
 |                       GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
@@ -68,7 +68,7 @@ Legend:
 |                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                          L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                       LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                LIGHTNING_INDEXER | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                LIGHTNING_INDEXER | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |                              LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |

examples/model-conversion/scripts/causal/convert-model.sh

@@ -47,7 +47,6 @@ CMD_ARGS+=("../../convert_hf_to_gguf.py" "--verbose")
 CMD_ARGS+=("${MODEL_PATH}")
 CMD_ARGS+=("--outfile" "${CONVERTED_MODEL}")
 CMD_ARGS+=("--outtype" "${TYPE}")
-CMD_ARGS+=("--model-name" "${MODEL_NAME}")
 [[ -n "$METADATA_OVERRIDE" ]] && CMD_ARGS+=("--metadata" "${METADATA_OVERRIDE}")
 [[ -n "$MMPROJ" ]] && CMD_ARGS+=("${MMPROJ}")
 

examples/model-conversion/scripts/embedding/convert-model.sh

@@ -31,7 +31,6 @@ python ../../convert_hf_to_gguf.py --verbose \
     ${EMBEDDING_MODEL_PATH} \
     --outfile ${CONVERTED_MODEL} \
     --outtype ${TYPE} \
-    --model-name ${MODEL_NAME} \
     ${SENTENCE_TRANSFORMERS}
 
 echo ""

examples/sycl/start-svr.sh

@@ -12,7 +12,6 @@ This script processes files with specified options.
 
 Options:
   -h, --help    Display this help message and exit.
-  -d, --device  <value>    Set SYCL devices (default: SYCL0).
   -c, --context <value>    Set context length. Bigger need more memory.
   -p, --promote <value>    Prompt to start generation with.
   -m, --model   <value>    Full model file path.
@@ -42,16 +41,10 @@ MODEL_FILE=../models/Qwen3.5-4B-Q4_0.gguf
 NGL=99
 CONTEXT=4096
 GGML_SYCL_DEVICE=-1
-SYCL_DEVICES="SYCL0"
 SPLIT_MODE=layer
 LOG_VERBOSE=3
 while [[ $# -gt 0 ]]; do
     case "$1" in
-        -d|--device)
-            SYCL_DEVICES="$2"
-            shift
-            shift
-            ;;
         -c|--context)
             CONTEXT=$2
             # Shift twice to consume both the option flag and its value
@@ -102,6 +95,8 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
+
+
 source /opt/intel/oneapi/setvars.sh
 
 #export GGML_SYCL_DEBUG=1
@@ -112,19 +107,17 @@ source /opt/intel/oneapi/setvars.sh
 export UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1
 echo "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=${UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS}"
 
-echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"
-
-
 if [ $GGML_SYCL_DEVICE -ne -1 ]; then
     echo "Use $GGML_SYCL_DEVICE as main GPU"
     #use signle GPU only
     GPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}"
+    echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"
 else
-    echo "Use Intel GPUs: ${SYCL_DEVICES}"
+    echo "Use all Intel GPUs, including iGPU & dGPU"
     GPUS_SETTING="-sm ${SPLIT_MODE}"
-fi
+ fi
 
-echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000"
-ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000
+echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE}  --mmap --host 0.0.0.0 --port 8000"
+ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap --host 0.0.0.0 --port 8000
 
 

examples/sycl/test.sh

@@ -12,7 +12,6 @@ This script processes files with specified options.
 
 Options:
   -h, --help    Display this help message and exit.
-  -d, --device  <value>    Set SYCL devices (default: SYCL0).
   -c, --context <value>    Set context length. Bigger need more memory.
   -p, --promote <value>    Prompt to start generation with.
   -m, --model   <value>    Full model file path.
@@ -43,16 +42,10 @@ MODEL_FILE=../models/llama-2-7b.Q4_0.gguf
 NGL=99
 CONTEXT=4096
 GGML_SYCL_DEVICE=-1
-SYCL_DEVICES="SYCL0"
 SPLIT_MODE=layer
 LOG_VERBOSE=3
 while [[ $# -gt 0 ]]; do
     case "$1" in
-        -d|--device)
-            SYCL_DEVICES="$2"
-            shift
-            shift
-            ;;
         -c|--context)
             CONTEXT=$2
             # Shift twice to consume both the option flag and its value
@@ -122,17 +115,16 @@ source /opt/intel/oneapi/setvars.sh
 export UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1
 echo "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=${UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS}"
 
-echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"
-
 if [ $GGML_SYCL_DEVICE -ne -1 ]; then
     echo "Use $GGML_SYCL_DEVICE as main GPU"
     #use signle GPU only
     GPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}"
+    echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"
 else
-    echo "Use Intel GPUs: ${SYCL_DEVICES}"
+    echo "Use all Intel GPUs, including iGPU & dGPU"
     GPUS_SETTING="-sm ${SPLIT_MODE}"
  fi
 
-echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap "
-ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap
+echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE}  --mmap "
+ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap
 

examples/sycl/win-start-svr.bat

@@ -13,7 +13,6 @@ set "MODEL_FILE=..\models\Qwen3.5-4B-Q4_0.gguf"
 set "NGL=99"
 set "CONTEXT=4096"
 set "GGML_SYCL_DEVICE=-1"
-set "SYCL_DEVICES=SYCL0"
 set "SPLIT_MODE=layer"
 set "LOG_VERBOSE=3"
 
@@ -37,21 +36,6 @@ if /I "%~1"=="--context" (
   goto parse_args
 )
 
-if /I "%~1"=="-d" (
-  if "%~2"=="" goto missing_value
-  set "SYCL_DEVICES=%~2"
-  shift
-  shift
-  goto parse_args
-)
-if /I "%~1"=="--device" (
-  if "%~2"=="" goto missing_value
-  set "SYCL_DEVICES=%~2"
-  shift
-  shift
-  goto parse_args
-)
-
 if /I "%~1"=="-m" (
   if "%~2"=="" goto missing_value
   set "MODEL_FILE=%~2"
@@ -146,7 +130,6 @@ echo This script processes files with specified options.
 echo.
 echo Options:
 echo   -h, --help    Display this help message and exit.
-echo   -d, --device ^<value^>    Set SYCL devices (default: SYCL0).
 echo   -c, --context ^<value^>    Set context length. Bigger need more memory.
 echo   -m, --model   ^<value^>    Full model file path.
 echo   -mg,--main-gpu ^<value^>   Set main GPU ID (0 - n) for single GPU mode.
@@ -177,20 +160,19 @@ REM Support malloc device memory more than 4GB.
 set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"
 echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%
 
-echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%
-
 if not "%GGML_SYCL_DEVICE%"=="-1" (
   echo Use %GGML_SYCL_DEVICE% as main GPU
   REM Use single GPU only.
   set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"
-  ) else (
-  echo Use Intel GPUs: %SYCL_DEVICES%
+  echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%
+) else (
+  echo Use all Intel GPUs, including iGPU ^& dGPU
   set "GPUS_SETTING=-sm %SPLIT_MODE%"
 )
 
-echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap --host 0.0.0.0 --port 8000
+echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000
 set "ZES_ENABLE_SYSMAN=1"
-%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap --host 0.0.0.0 --port 8000
+%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000
 
 endlocal
 

examples/sycl/win-test.bat

@@ -19,7 +19,6 @@ set "MODEL_FILE=..\models\llama-2-7b.Q4_0.gguf"
 set "NGL=99"
 set "CONTEXT=4096"
 set "GGML_SYCL_DEVICE=-1"
-set "SYCL_DEVICES=SYCL0"
 set "SPLIT_MODE=layer"
 set "LOG_VERBOSE=3"
 
@@ -43,21 +42,6 @@ if /I "%~1"=="--context" (
   goto parse_args
 )
 
-if /I "%~1"=="-d" (
-  if "%~2"=="" goto missing_value
-  set "SYCL_DEVICES=%~2"
-  shift
-  shift
-  goto parse_args
-)
-if /I "%~1"=="--device" (
-  if "%~2"=="" goto missing_value
-  set "SYCL_DEVICES=%~2"
-  shift
-  shift
-  goto parse_args
-)
-
 if /I "%~1"=="-p" (
   if "%~2"=="" goto missing_value
   set "INPUT_PROMPT=%~2"
@@ -167,7 +151,6 @@ echo This script processes files with specified options.
 echo.
 echo Options:
 echo   -h, --help    Display this help message and exit.
-echo   -d, --device ^<value^>    Set SYCL devices (default: SYCL0).
 echo   -c, --context ^<value^>    Set context length. Bigger need more memory.
 echo   -p, --promote ^<value^>    Prompt to start generation with.
 echo   -m, --model   ^<value^>    Full model file path.
@@ -199,21 +182,19 @@ REM Support malloc device memory more than 4GB.
 set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"
 echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%
 
-echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%
-
 if not "%GGML_SYCL_DEVICE%"=="-1" (
   echo Use %GGML_SYCL_DEVICE% as main GPU
   REM Use single GPU only.
   set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"
-  )
-else (
-  echo Use Intel GPUs: %SYCL_DEVICES%
+  echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%
+) else (
+  echo Use all Intel GPUs, including iGPU ^& dGPU
   set "GPUS_SETTING=-sm %SPLIT_MODE%"
 )
 
-echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap
+echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap
 set "ZES_ENABLE_SYSMAN=1"
-%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap
+%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap
 
 endlocal
 

ggml/CMakeLists.txt

@@ -4,8 +4,8 @@ project("ggml" C CXX ASM)
 
 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 19)
-set(GGML_VERSION_PATCH 0)
+set(GGML_VERSION_MINOR 18)
+set(GGML_VERSION_PATCH 1)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")

ggml/include/ggml-rpc.h

@@ -6,8 +6,8 @@
 extern "C" {
 #endif
 
-#define RPC_PROTO_MAJOR_VERSION    5
-#define RPC_PROTO_MINOR_VERSION    0
+#define RPC_PROTO_MAJOR_VERSION    6
+#define RPC_PROTO_MINOR_VERSION    1
 #define RPC_PROTO_PATCH_VERSION    0
 
 #ifdef  __cplusplus

ggml/include/ggml.h

@@ -2788,12 +2788,6 @@ extern "C" {
             struct ggml_cgraph * cgraph,
             struct ggml_tensor * tensor);
 
-    // add the tensor and its parents to the graph without marking them for compute
-    // the flag is set later, when the tensor is reached from a node that computes
-    GGML_API void ggml_build_forward_order(
-            struct ggml_cgraph * cgraph,
-            struct ggml_tensor * tensor);
-
     GGML_API void ggml_build_backward_expand(
         struct ggml_context *  ctx,        // context for gradient computation
         struct ggml_cgraph  *  cgraph,

ggml/src/ggml-backend-meta.cpp

@@ -590,7 +590,18 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
             GGML_ASSERT(split_states_equal(src_ss[0], src_ss[1]));
             return {assume_sync ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_PARTIAL, {0}, {1}, 1};
         }
-        GGML_ABORT("fatal error");
+        if (src_ss[0].axis == src_ss[1].axis && src_ss[0].axis >= GGML_BACKEND_SPLIT_AXIS_2 &&
+                src_ss[0].axis < GGML_MAX_DIMS) {
+            GGML_ASSERT(split_states_equal(src_ss[0], src_ss[1]));
+            return src_ss[0];
+        }
+        // batched matmul with the batches split across devices and a replicated activation
+        if (src_ss[0].axis >= GGML_BACKEND_SPLIT_AXIS_2 && src_ss[0].axis < GGML_MAX_DIMS &&
+                src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
+            return src_ss[0];
+        }
+        GGML_ABORT("unsupported mul_mat split states: node=%s src0=%s axis=%d src1=%s axis=%d",
+            tensor->name, tensor->src[0]->name, (int) src_ss[0].axis, tensor->src[1]->name, (int) src_ss[1].axis);
         //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1};
     };
 
@@ -745,14 +756,33 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
     };
 
     auto handle_flash_attn_ext = [&](const std::vector<ggml_backend_meta_split_state> & src_ss) -> ggml_backend_meta_split_state {
-        GGML_ASSERT(                             src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_2);
-        GGML_ASSERT(                             src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_2);
-        GGML_ASSERT(                             src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_2);
-        GGML_ASSERT(tensor->src[4] == nullptr || src_ss[3].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+        GGML_ASSERT(tensor->src[3] == nullptr || src_ss[3].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+
+        if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
+            GGML_ASSERT(src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+            GGML_ASSERT(src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+            GGML_ASSERT(tensor->src[4] == nullptr || src_ss[4].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+            return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1};
+        }
+
+        GGML_ASSERT(src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_2);
+        const bool kv_split = src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_2 &&
+                src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_2;
+        const bool kv_mirrored = src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED &&
+                src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED;
+        GGML_ASSERT(kv_split || kv_mirrored);
         GGML_ASSERT(tensor->src[4] == nullptr || src_ss[4].axis == GGML_BACKEND_SPLIT_AXIS_0);
         return {GGML_BACKEND_SPLIT_AXIS_1, {0}, {1}, 1};
     };
 
+    auto handle_lightning_indexer = [&](
+            const std::vector<ggml_backend_meta_split_state> & src_ss) -> ggml_backend_meta_split_state {
+        for (size_t i = 0; i < 4; i++) {
+            GGML_ASSERT(src_ss[i].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+        }
+        return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1};
+    };
+
     auto handle_ssm_conv = [&](const std::vector<ggml_backend_meta_split_state> & src_ss) -> ggml_backend_meta_split_state {
         if (src_ss[0].axis == src_ss[1].axis) {
             if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_0) {
@@ -817,7 +847,12 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
         ggml_backend_meta_split_state split_state;
         switch (tensor->op) {
             case GGML_OP_NONE: {
-                split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1};
+                if (tensor->view_src != nullptr) {
+                    // full-tensor view created with ggml_view_tensor, transparent for the split state
+                    split_state = ggml_backend_meta_get_split_state(stc, tensor->view_src, assume_sync);
+                } else {
+                    split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1};
+                }
             } break;
             case GGML_OP_DUP: {
                 split_state = handle_generic(src_ss, /*scalar_only =*/ true);
@@ -920,7 +955,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
                 split_state = handle_rope(src_ss);
             } break;
             case GGML_OP_ROPE_BACK: {
-                split_state = handle_generic(src_ss, /*scalar_only =*/ true);
+                split_state = handle_rope(src_ss);
             } break;
             case GGML_OP_CLAMP: {
                 split_state = handle_generic(src_ss, /*scalar_only =*/ false);
@@ -984,6 +1019,9 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
             case GGML_OP_GATED_DELTA_NET: {
                 split_state = handle_gated_delta_net(src_ss);
             } break;
+            case GGML_OP_LIGHTNING_INDEXER: {
+                split_state = handle_lightning_indexer(src_ss);
+            } break;
             case GGML_OP_DSV4_HC_COMB:
             case GGML_OP_DSV4_HC_PRE:
             case GGML_OP_DSV4_HC_POST: {
@@ -1068,13 +1106,14 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
         if (buf_ctx->debug > 0) {
             std::string srcs_info;
             for (size_t i = 0; i < GGML_MAX_SRC; i++) {
-                if (tensor->src[i] == nullptr) {
+                if (tensor->src[i] == nullptr || tensor->src[i] == tensor) {
                     continue;
                 }
                 if (!srcs_info.empty()) {
                     srcs_info += ", ";
                 }
-                const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor->src[0], true);
+                const ggml_backend_meta_split_state split_state =
+                        ggml_backend_meta_get_split_state(tensor->src[i], true);
                 GGML_ASSERT(split_state.n_segments == 1);
                 const char * axis_name = ggml_backend_meta_split_axis_name(split_state.axis);
                 std::string ne_info;
@@ -1253,6 +1292,108 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer
     return ggml_backend_meta_buffer_init_tensor_impl(buf_ctx->get_simple_tensor_container(tensor), tensor);
 }
 
+static void ggml_backend_meta_buffer_memset_tensor(
+        ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+    const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
+    const ggml_backend_meta_split_state split_state =
+            ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false);
+    GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+
+    if (split_state.n_segments != 1 || split_state.nr[0] != 1) {
+        GGML_ASSERT(split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS);
+        GGML_ASSERT(split_state.nr[0] != 0);
+        GGML_ASSERT(tensor->ne[3] == 1);
+
+        std::vector<size_t> simple_offsets(n_bufs, 0);
+        if (split_state.axis == GGML_BACKEND_SPLIT_AXIS_0) {
+            GGML_ASSERT(tensor->ne[2] == 1);
+
+            const size_t row_stride = tensor->nb[1];
+            GGML_ASSERT(offset % row_stride == 0);
+            GGML_ASSERT(size   % row_stride == 0);
+            const int64_t row_start = offset / row_stride;
+            const int64_t row_count = size   / row_stride;
+            GGML_ASSERT(row_start + row_count <= tensor->ne[1]);
+
+            const int64_t blck_size = ggml_blck_size(tensor->type);
+            for (size_t s = 0; s < split_state.n_segments; s++) {
+                for (size_t r = 0; r < split_state.nr[s]; r++) {
+                    for (size_t j = 0; j < n_bufs; j++) {
+                        ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j);
+                        GGML_ASSERT(split_state.ne[s*n_bufs + j] % blck_size == 0);
+                        const size_t nbytes = split_state.ne[s*n_bufs + j]/blck_size * tensor->nb[0];
+                        for (int64_t row = 0; row < row_count; row++) {
+                            ggml_backend_tensor_memset(simple_tensor, value,
+                                    simple_offsets[j] + (row_start + row)*simple_tensor->nb[1], nbytes);
+                        }
+                        simple_offsets[j] += nbytes;
+                    }
+                }
+            }
+            return;
+        }
+
+        GGML_ASSERT(split_state.axis == GGML_BACKEND_SPLIT_AXIS_1);
+
+        const size_t row_stride = tensor->nb[2];
+        GGML_ASSERT(offset % row_stride == 0);
+        GGML_ASSERT(size   % row_stride == 0);
+        const int64_t row_start = offset / row_stride;
+        const int64_t row_count = size   / row_stride;
+        GGML_ASSERT(row_start + row_count <= tensor->ne[2]);
+
+        for (size_t s = 0; s < split_state.n_segments; s++) {
+            for (size_t r = 0; r < split_state.nr[s]; r++) {
+                for (size_t j = 0; j < n_bufs; j++) {
+                    ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j);
+                    const size_t nbytes = split_state.ne[s*n_bufs + j] * tensor->nb[1];
+                    for (int64_t row = 0; row < row_count; row++) {
+                        ggml_backend_tensor_memset(simple_tensor, value,
+                                simple_offsets[j] + (row_start + row)*simple_tensor->nb[2], nbytes);
+                    }
+                    simple_offsets[j] += nbytes;
+                }
+            }
+        }
+        return;
+    }
+
+    switch (split_state.axis) {
+        case GGML_BACKEND_SPLIT_AXIS_0:
+        case GGML_BACKEND_SPLIT_AXIS_1:
+        case GGML_BACKEND_SPLIT_AXIS_2: {
+            const size_t chunk_size_full = tensor->nb[split_state.axis + 1];
+            GGML_ASSERT(offset % chunk_size_full == 0);
+            GGML_ASSERT(size   % chunk_size_full == 0);
+            const int64_t i_start =  offset        / chunk_size_full;
+            const int64_t i_stop  = (offset + size) / chunk_size_full;
+            for (size_t j = 0; j < n_bufs; j++) {
+                ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j);
+                const size_t chunk_size = simple_tensor->nb[split_state.axis + 1];
+                if (chunk_size == 0) {
+                    continue;
+                }
+                for (int64_t i = i_start; i < i_stop; i++) {
+                    ggml_backend_tensor_memset(simple_tensor, value, i*chunk_size, chunk_size);
+                }
+            }
+        } break;
+        case GGML_BACKEND_SPLIT_AXIS_PARTIAL: {
+            GGML_ASSERT(value == 0);
+            [[fallthrough]];
+        }
+        case GGML_BACKEND_SPLIT_AXIS_MIRRORED: {
+            for (size_t j = 0; j < n_bufs; j++) {
+                ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j);
+                ggml_backend_tensor_memset(simple_tensor, value, offset, size);
+            }
+        } break;
+        default: {
+            GGML_ABORT("fatal error");
+        }
+    }
+}
+
 static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
     const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
     const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false);
@@ -1486,7 +1627,7 @@ static const ggml_backend_buffer_i ggml_backend_meta_buffer_iface = {
     /* .free_buffer     = */ ggml_backend_meta_buffer_free_buffer,
     /* .get_base        = */ ggml_backend_meta_buffer_get_base,
     /* .init_tensor     = */ ggml_backend_meta_buffer_init_tensor,
-    /* .memset_tensor   = */ nullptr, // TODO implement
+    /* .memset_tensor   = */ ggml_backend_meta_buffer_memset_tensor,
     /* .set_tensor      = */ ggml_backend_meta_buffer_set_tensor,
     /* .get_tensor      = */ ggml_backend_meta_buffer_get_tensor,
     /* .set_tensor_2d   = */ nullptr,
@@ -2043,6 +2184,14 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
                 cgraph_ij->uid = ggml_graph_next_uid();
             }
         }
+
+        // Aux graph contents are rewritten on every compute but are identical across calls while the subgraphs are reused,
+        // so they can get stable uids on rebuild. Only safe without a comm backend, where the fallback usage is deterministic.
+        if (backend_ctx->comm_ctx == nullptr) {
+            for (ggml_cgraph * cgraph_aux : backend_ctx->cgraphs_aux) {
+                cgraph_aux->uid = ggml_graph_next_uid();
+            }
+        }
     }
 
     size_t iga = 0; // i graph aux

ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp

@@ -8,22 +8,6 @@
 #include <sys/sysctl.h>
 #endif
 
-#if !defined(HWCAP_FPHP)
-#define HWCAP_FPHP (1 << 9)
-#endif
-
-#if !defined(HWCAP_ASIMDHP)
-#define HWCAP_ASIMDHP (1 << 10)
-#endif
-
-#if !defined(HWCAP_ASIMDDP)
-#define HWCAP_ASIMDDP (1 << 20)
-#endif
-
-#if !defined(HWCAP_SVE)
-#define HWCAP_SVE (1 << 22)
-#endif
-
 #if !defined(HWCAP2_SVE2)
 #define HWCAP2_SVE2 (1 << 1)
 #endif
@@ -39,7 +23,7 @@
 struct aarch64_features {
     // has_neon not needed, aarch64 has NEON guaranteed
     bool has_dotprod     = false;
-    bool has_fp16        = false;
+    bool has_fp16_va     = false;
     bool has_sve         = false;
     bool has_sve2        = false;
     bool has_i8mm        = false;
@@ -52,7 +36,7 @@ struct aarch64_features {
         uint32_t hwcap2 = getauxval(AT_HWCAP2);
 
         has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
-        has_fp16    = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);
+        has_fp16_va = !!(hwcap & HWCAP_FPHP);
         has_sve     = !!(hwcap & HWCAP_SVE);
         has_sve2    = !!(hwcap2 & HWCAP2_SVE2);
         has_i8mm    = !!(hwcap2 & HWCAP2_I8MM);
@@ -91,7 +75,7 @@ static int ggml_backend_cpu_aarch64_score() {
     score += 1<<1;
 #endif
 #ifdef GGML_USE_FP16_VECTOR_ARITHMETIC
-    if (!af.has_fp16) { return 0; }
+    if (!af.has_fp16_va) { return 0; }
     score += 1<<2;
 #endif
 #ifdef GGML_USE_SVE

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -4033,11 +4033,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud
                     continue;
                 }
 #ifndef NDEBUG
-                // On integrated GPUs (APUs, e.g. RDNA3.5) the scheduler may place a
-                // node's output on the host-visible buffer, which the compute path
-                // handles. Allow that here, mirroring the src-tensor check below.
-                assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) ||
-                       (integrated && ggml_backend_buft_is_cuda_host(node->buffer->buft)));
+                assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));
                 for (int j = 0; j < GGML_MAX_SRC; j++) {
                     if (node->src[j] != nullptr) {
                         assert(node->src[j]->buffer);
@@ -5209,7 +5205,6 @@ static bool ggml_backend_cuda_device_offload_op(ggml_backend_dev_t dev, const gg
 
 static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_t dev) {
 #ifdef GGML_CUDA_NO_PEER_COPY
-    GGML_UNUSED(dev);
     return nullptr;
 #else
     ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *)dev->context;

ggml/src/ggml-cuda/quantize.cu

@@ -8,6 +8,7 @@ struct __builtin_align__(32) float8 {
     float x; float y; float z; float w;
     float p; float q; float r; float s;
 };
+#endif
 
 #if CUDART_VERSION >= 12080
 static __device__ __forceinline__ float nvfp4_native_scale_error(
@@ -48,7 +49,6 @@ static __device__ __forceinline__ float nvfp4_native_scale_error(
     return err;
 }
 #endif // CUDART_VERSION >= 12080
-#endif // defined(BLACKWELL_MMA_AVAILABLE)
 
 __launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1)
 static __global__ void quantize_q8_1(

ggml/src/ggml-metal/ggml-metal.metal

@@ -11328,8 +11328,8 @@ kernel void kernel_lightning_indexer(
     const int i_kv_0   = tgpig.x*NK;            // first key of this threadgroup
     const int i_kv     = i_kv_0 + sgitg*NKPSG;  // first key of this simdgroup
 
-    threadgroup half sk[NK * DK16 * 16];
-    threadgroup half4x4 * sk4x4 = (threadgroup half4x4 *) sk;
+    threadgroup half4x4   sk4x4[NK*DK16];
+    threadgroup half    * sk = (threadgroup half *) sk4x4;
 
     for (short i = tiitg; i < NK*DK16; i += NTG) {
         const short ik  = i/DK16;

ggml/src/ggml-sycl/common.hpp

@@ -1022,20 +1022,9 @@ static T block_reduce(T val, T * shared_vals, int block_size_template) {
 }
 
 static __dpct_inline__ float ggml_sycl_ue4m3_to_fp32(uint8_t x) {
-    // UE4M3 is unsigned: 4 exp bits (bias 7), 3 mantissa bits, no sign, no NaN.
-    // exp == 0xF is a valid exponent (256-448 range), not NaN.
-    if (x == 0 || x == 0x7F) {
-        return 0.0f;
-    }
-    const int exp = (x >> 3) & 0xF;
-    const int man = x & 0x7;
-    float raw;
-    if (exp == 0) {
-        raw = man * (1.0f / 8.0f) * sycl::pow(2.0f, -6.0f);
-    } else {
-        raw = (1.0f + man / 8.0f) * sycl::pow(2.0f, (float) exp - 7.0f);
-    }
-    return raw * 0.5f;
+    const uint32_t bits = x * (x != 0x7F && x != 0xFF);
+    const __nv_fp8_e4m3 xf = *reinterpret_cast<const __nv_fp8_e4m3 *>(&bits);
+    return static_cast<float>(xf) / 2;
 }
 
 #endif // GGML_SYCL_COMMON_HPP

ggml/src/ggml-sycl/dsv4-hc.cpp

@@ -1,280 +0,0 @@
-#include "ggml-impl.h"
-#include "dsv4-hc.hpp"
-
-#include <cmath>
-
-static constexpr int DSV4_HC = 4;
-
-static void dsv4_hc_pre_f32_sycl(
-        const float * x, const float * weights, float * dst,
-        int64_t n_embd, int64_t hc, int64_t n_tokens,
-        int64_t sx0, int64_t sx1, int64_t sx2,
-        int64_t sw0, int64_t sw1,
-        int64_t sd0, int64_t sd1,
-        queue_ptr stream) {
-    const int64_t nr = n_embd * n_tokens;
-    const int64_t block_size = 256;
-    const int64_t num_blocks = (nr + block_size - 1) / block_size;
-
-    stream->parallel_for(
-        sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)),
-        [=](sycl::nd_item<1> item) {
-            const int64_t ir = item.get_global_id(0);
-            if (ir >= nr) {
-                return;
-            }
-
-            const int64_t i0 = ir % n_embd;
-            const int64_t it = ir / n_embd;
-
-            float sum = x[i0*sx0 + it*sx2] * weights[it*sw1];
-            for (int64_t ih = 1; ih < hc; ++ih) {
-                const float xv = x[i0*sx0 + ih*sx1 + it*sx2];
-                const float wv = weights[ih*sw0 + it*sw1];
-                sum += xv * wv;
-            }
-
-            dst[i0*sd0 + it*sd1] = sum;
-        });
-}
-
-static void dsv4_hc_comb_norm_cols(float * comb, float eps) {
-    for (int idst = 0; idst < DSV4_HC; ++idst) {
-        float sum = eps;
-        for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
-            sum += comb[idst + DSV4_HC*isrc];
-        }
-
-        const float inv_sum = 1.0f / sum;
-        for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
-            comb[idst + DSV4_HC*isrc] *= inv_sum;
-        }
-    }
-}
-
-static void dsv4_hc_comb_norm_rows(float * comb, float eps) {
-    for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
-        float sum = eps;
-        for (int idst = 0; idst < DSV4_HC; ++idst) {
-            sum += comb[idst + DSV4_HC*isrc];
-        }
-
-        const float inv_sum = 1.0f / sum;
-        for (int idst = 0; idst < DSV4_HC; ++idst) {
-            comb[idst + DSV4_HC*isrc] *= inv_sum;
-        }
-    }
-}
-
-static void dsv4_hc_comb_f32_sycl(
-        const float * mixes,
-        const float * scale,
-        const float * base,
-        float * dst,
-        int64_t n_tokens,
-        int64_t sm0,
-        int64_t sm1,
-        int64_t ss0,
-        int64_t sb0,
-        int64_t sd0,
-        int64_t sd1,
-        int64_t sd2,
-        float eps,
-        int32_t n_iter,
-        queue_ptr stream) {
-    constexpr int comb_offset = 2*DSV4_HC;
-
-    const int64_t block_size = 256;
-    const int64_t num_blocks = (n_tokens + block_size - 1) / block_size;
-
-    stream->parallel_for(
-        sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)),
-        [=](sycl::nd_item<1> item_ct1) {
-            const int64_t it = item_ct1.get_global_id(0);
-
-            if (it >= n_tokens) {
-                return;
-            }
-
-            const float scale_comb = scale[2*ss0];
-            float comb[DSV4_HC*DSV4_HC];
-
-            for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
-                float max = -INFINITY;
-                for (int idst = 0; idst < DSV4_HC; ++idst) {
-                    const int idx = idst + DSV4_HC*isrc;
-                    const float v = mixes[(comb_offset + idx)*sm0 + it*sm1] * scale_comb + base[(comb_offset + idx)*sb0];
-                    comb[idx] = v;
-                    max = fmaxf(max, v);
-                }
-
-                float sum = 0.0f;
-                for (int idst = 0; idst < DSV4_HC; ++idst) {
-                    const int idx = idst + DSV4_HC*isrc;
-                    const float v = expf(comb[idx] - max);
-                    comb[idx] = v;
-                    sum += v;
-                }
-
-                const float inv_sum = 1.0f / sum;
-                for (int idst = 0; idst < DSV4_HC; ++idst) {
-                    const int idx = idst + DSV4_HC*isrc;
-                    comb[idx] = comb[idx] * inv_sum + eps;
-                }
-            }
-
-            dsv4_hc_comb_norm_cols(comb, eps);
-            for (int32_t i = 1; i < n_iter; ++i) {
-                dsv4_hc_comb_norm_rows(comb, eps);
-                dsv4_hc_comb_norm_cols(comb, eps);
-            }
-
-            for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
-                for (int idst = 0; idst < DSV4_HC; ++idst) {
-                    const int idx = idst + DSV4_HC*isrc;
-                    dst[idst*sd0 + isrc*sd1 + it*sd2] = comb[idx];
-                }
-            }
-        });
-}
-
-static void dsv4_hc_post_f32_sycl(
-        const float * x, const float * residual, const float * post, const float * comb, float * dst,
-        int64_t n_embd, int64_t hc, int64_t n_tokens,
-        int64_t sx0, int64_t sx1,
-        int64_t sr0, int64_t sr1, int64_t sr2,
-        int64_t sp0, int64_t sp1,
-        int64_t sc0, int64_t sc1, int64_t sc2,
-        int64_t sd0, int64_t sd1, int64_t sd2,
-        queue_ptr stream) {
-    const int64_t nr = n_embd * hc * n_tokens;
-    const int64_t block_size = 256;
-    const int64_t num_blocks = (nr + block_size - 1) / block_size;
-
-    stream->parallel_for(
-        sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)),
-        [=](sycl::nd_item<1> item) {
-            const int64_t ir = item.get_global_id(0);
-            if (ir >= nr) {
-                return;
-            }
-
-            const int64_t i0   = ir % n_embd;
-            const int64_t idst = (ir / n_embd) % hc;
-            const int64_t it   = ir / (n_embd * hc);
-
-            float sum = x[i0*sx0 + it*sx1] * post[idst*sp0 + it*sp1];
-            for (int64_t isrc = 0; isrc < hc; ++isrc) {
-                sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2];
-            }
-
-            dst[i0*sd0 + idst*sd1 + it*sd2] = sum;
-        });
-}
-
-void ggml_sycl_op_dsv4_hc_pre(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
-    const ggml_tensor * x       = dst->src[0];
-    const ggml_tensor * weights = dst->src[1];
-
-    GGML_ASSERT(x->type == GGML_TYPE_F32);
-    GGML_ASSERT(weights->type == GGML_TYPE_F32);
-    GGML_ASSERT(dst->type == GGML_TYPE_F32);
-
-    GGML_TENSOR_LOCALS(size_t, nbx, x,       nb);
-    GGML_TENSOR_LOCALS(size_t, nbw, weights, nb);
-    GGML_TENSOR_LOCALS(size_t, nbd, dst,     nb);
-
-    const int64_t n_embd   = x->ne[0];
-    const int64_t hc       = x->ne[1];
-    const int64_t n_tokens = x->ne[2];
-
-    queue_ptr stream = ctx.stream();
-
-    dsv4_hc_pre_f32_sycl(
-            (const float *) x->data, (const float *) weights->data, (float *) dst->data,
-            n_embd, hc, n_tokens,
-            nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float),
-            nbw0 / sizeof(float), nbw1 / sizeof(float),
-            nbd0 / sizeof(float), nbd1 / sizeof(float),
-            stream);
-}
-
-void ggml_sycl_op_dsv4_hc_comb(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3);
-
-    const ggml_tensor * mixes = dst->src[0];
-    const ggml_tensor * scale = dst->src[1];
-    const ggml_tensor * base  = dst->src[2];
-
-    GGML_ASSERT(mixes->type == GGML_TYPE_F32);
-    GGML_ASSERT(scale->type == GGML_TYPE_F32);
-    GGML_ASSERT(base->type == GGML_TYPE_F32);
-    GGML_ASSERT(dst->type == GGML_TYPE_F32);
-
-    constexpr int64_t hc_mix_dim = (2 + DSV4_HC)*DSV4_HC;
-
-    GGML_ASSERT(mixes->ne[0] == hc_mix_dim);
-    GGML_ASSERT(dst->ne[0] == DSV4_HC);
-    GGML_ASSERT(dst->ne[1] == DSV4_HC);
-    GGML_ASSERT(dst->ne[2] == mixes->ne[1]);
-    GGML_ASSERT(scale->ne[0] >= 3);
-    GGML_ASSERT(base->ne[0] == hc_mix_dim);
-
-    GGML_TENSOR_LOCALS(size_t, nbm, mixes, nb);
-    GGML_TENSOR_LOCALS(size_t, nbs, scale, nb);
-    GGML_TENSOR_LOCALS(size_t, nbb, base,  nb);
-    GGML_TENSOR_LOCALS(size_t, nbd, dst,   nb);
-
-    const int64_t n_tokens = mixes->ne[1];
-    const float eps = ggml_get_op_params_f32(dst, 0);
-    const int32_t n_iter = ggml_get_op_params_i32(dst, 1);
-
-    queue_ptr stream = ctx.stream();
-
-    dsv4_hc_comb_f32_sycl(
-            (const float *) mixes->data, (const float *) scale->data, (const float *) base->data, (float *) dst->data,
-            n_tokens,
-            nbm0 / sizeof(float), nbm1 / sizeof(float),
-            nbs0 / sizeof(float),
-            nbb0 / sizeof(float),
-            nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float),
-            eps, n_iter, stream);
-}
-
-void ggml_sycl_op_dsv4_hc_post(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/4);
-    const ggml_tensor * x        = dst->src[0];
-    const ggml_tensor * residual = dst->src[1];
-    const ggml_tensor * post     = dst->src[2];
-    const ggml_tensor * comb     = dst->src[3];
-
-    GGML_ASSERT(x->type == GGML_TYPE_F32);
-    GGML_ASSERT(residual->type == GGML_TYPE_F32);
-    GGML_ASSERT(post->type == GGML_TYPE_F32);
-    GGML_ASSERT(comb->type == GGML_TYPE_F32);
-    GGML_ASSERT(dst->type == GGML_TYPE_F32);
-
-    GGML_TENSOR_LOCALS(size_t, nbx, x,        nb);
-    GGML_TENSOR_LOCALS(size_t, nbr, residual, nb);
-    GGML_TENSOR_LOCALS(size_t, nbp, post,     nb);
-    GGML_TENSOR_LOCALS(size_t, nbc, comb,     nb);
-    GGML_TENSOR_LOCALS(size_t, nbd, dst,      nb);
-
-    const int64_t n_embd   = x->ne[0];
-    const int64_t n_tokens = x->ne[1];
-    const int64_t hc       = residual->ne[1];
-
-    queue_ptr stream = ctx.stream();
-
-    dsv4_hc_post_f32_sycl(
-            (const float *) x->data, (const float *) residual->data,
-            (const float *) post->data, (const float *) comb->data, (float *) dst->data,
-            n_embd, hc, n_tokens,
-            nbx0 / sizeof(float), nbx1 / sizeof(float),
-            nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float),
-            nbp0 / sizeof(float), nbp1 / sizeof(float),
-            nbc0 / sizeof(float), nbc1 / sizeof(float), nbc2 / sizeof(float),
-            nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float),
-            stream);
-}

ggml/src/ggml-sycl/dsv4-hc.hpp

@@ -1,10 +0,0 @@
-#ifndef GGML_SYCL_DSV4_HC_HPP
-#define GGML_SYCL_DSV4_HC_HPP
-
-#include "common.hpp"
-
-void ggml_sycl_op_dsv4_hc_pre(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
-void ggml_sycl_op_dsv4_hc_comb(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
-void ggml_sycl_op_dsv4_hc_post(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
-
-#endif // GGML_SYCL_DSV4_HC_HPP

ggml/src/ggml-sycl/element_wise.cpp

@@ -420,31 +420,53 @@ static void clamp(const T * x, T * dst, const float min, const float max, const
     }
 }
 
-template<typename T, typename F>
-static void unary_gated_op_flat_kernel(const T * x, const T * g, T * dst, const uint64_t k, const sycl::nd_item<1> & item_ct1, F func) {
-    SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
-        dst[i] = func(x[i]) * g[i];
-    }
-}
-
-template<typename T, typename F>
-static void unary_gated_op_generic_kernel(
-        const T * x,
-        const T * g,
-        T * dst,
-        const uint64_t k,
-        const sycl::uint3 n_fd,
-        const uint64_t o0,
-        const uint64_t o1,
-        const sycl::nd_item<1> & item_ct1,
-        F func) {
-
-    // rows of n columns at strides o0 and o1: two halves of one fused tensor, or two tensors
+template<typename T>
+static void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
     SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
         const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
         const int64_t j0 = rc.x() * o0 + rc.y();
         const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
-        dst[i] = func(x[j0]) * g[j1];
+        dst[i] = op_gelu(x[j0]) * g[j1];
+    }
+}
+
+template<typename T>
+static void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
+    SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
+        const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
+        const int64_t j0 = rc.x() * o0 + rc.y();
+        const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
+        dst[i] = op_relu(x[j0]) * g[j1];
+    }
+}
+
+template<typename T>
+static void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
+    SYCL_GLOBAL_ID_LOOP(k, item_ct1)  {
+        const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
+        const int64_t j0 = rc.x() * o0 + rc.y();
+        const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
+        dst[i] = op_silu(x[j0]) * g[j1];
+    }
+}
+
+template<typename T>
+static void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
+    SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
+        const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
+        const int64_t j0 = rc.x() * o0 + rc.y();
+        const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
+        dst[i] = op_gelu_erf(x[j0]) * g[j1];
+    }
+}
+
+template<typename T>
+static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
+    SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
+        const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
+        const int64_t j0 = rc.x() * o0 + rc.y();
+        const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
+        dst[i] = op_gelu_quick(x[j0]) * g[j1];
     }
 }
 
@@ -648,35 +670,6 @@ static inline void ggml_sycl_op_unary(
         });
 }
 
-template<typename F>
-static inline void ggml_sycl_op_unary_gated(
-        ggml_backend_sycl_context & ctx, ggml_tensor * dst, F func) {
-
-    dispatch_ggml_sycl_op_fused_glu(ctx, dst,
-        [func](const auto * x_ptr, const auto * g_ptr, auto * dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
-
-            const uint32_t num_blocks = (uint32_t) ceil_div(k, SYCL_GLU_BLOCK_SIZE);
-            const sycl::nd_range<1> launch_range(num_blocks * sycl::range<1>(SYCL_GLU_BLOCK_SIZE),
-                                                 sycl::range<1>(SYCL_GLU_BLOCK_SIZE));
-
-            // o0 == n and o1 == n make the index math the identity, so index flat
-            // note: not ggml_is_contiguous - a fused [gate|up] src0 is contiguous with o0 == 2n
-            if (o0 == n && o1 == n) {
-                main_stream->parallel_for(launch_range,
-                    [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                        unary_gated_op_flat_kernel(x_ptr, g_ptr, dst_ptr, k, item_ct1, func);
-                    });
-            } else {
-                // launch-invariant divisor, and only this path needs it
-                const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
-                main_stream->parallel_for(launch_range,
-                    [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                        unary_gated_op_generic_kernel(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1, func);
-                    });
-            }
-        });
-}
-
 
 static inline void ggml_sycl_op_arange(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     GGML_ASSERT(dst->type == GGML_TYPE_F32);
@@ -974,21 +967,42 @@ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor
 }
 
 static inline void ggml_sycl_op_geglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {
-        return op_gelu(x);
-    });
+    ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
+        [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
+            const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
+            const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
+            main_stream->parallel_for(
+                    sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
+                    sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
+            });
+        });
 }
 
 static inline void ggml_sycl_op_reglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {
-        return op_relu(x);
-    });
+    ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
+        [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
+            const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_RELU_BLOCK_SIZE); // Using RELU block size for reglu
+            const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
+            main_stream->parallel_for(
+                    sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_RELU_BLOCK_SIZE)),
+                    sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
+            });
+        });
 }
 
 static inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {
-        return op_silu(x);
-    });
+    ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
+        [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
+            const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_SILU_BLOCK_SIZE); // Using SILU block size for swiglu
+            const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
+            main_stream->parallel_for(
+                    sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_SILU_BLOCK_SIZE)),
+                    sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
+            });
+        });
 }
 
 __dpct_inline__ float ggml_sycl_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) {
@@ -1083,15 +1097,29 @@ void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst)
 }
 
 static inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {
-        return op_gelu_erf(x);
-    });
+    ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
+        [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
+            const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
+            const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
+            main_stream->parallel_for(
+                    sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
+                    sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
+            });
+        });
 }
 
 static inline void ggml_sycl_op_geglu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) {
-        return op_gelu_quick(x);
-    });
+    ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
+        [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
+            const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
+            const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
+            main_stream->parallel_for(
+                    sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
+                    sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
+            });
+        });
 }
 
 

ggml/src/ggml-sycl/fattn-vec.hpp

@@ -73,7 +73,6 @@ static void flash_attn_ext_vec(const char* __restrict__ Q,
                         const int32_t nb31,
                         const int32_t nb32,
                         const int64_t nb33) {
-
 #ifdef SYCL_FLASH_ATTN
     // Skip unused kernel variants for faster compilation:
 
@@ -470,6 +469,7 @@ static void flash_attn_ext_vec(const char* __restrict__ Q,
         }
     }
 
+
     item_ct1.barrier(sycl::access::fence_space::local_space);
 
 #pragma unroll
@@ -591,24 +591,22 @@ void ggml_sycl_flash_attn_ext_vec_case_impl(ggml_backend_sycl_context & ctx, ggm
 
     const auto arch = ggml_sycl_info().devices[ctx.device].hw_info.arch;
     const int nthreads = ggml_sycl_fattn_vec_get_nthreads_device(arch);
-    if constexpr (D <= 256) {
-        if (nthreads == 256) {
-            constexpr int nthreads_hw = 256;
-            constexpr int nwarps = nthreads_hw / warp_size;
-            launch_fattn<D, cols_per_block, 1,
-                         flash_attn_ext_vec<D, cols_per_block, type_K, type_V,
-                                            use_logit_softcap, warp_size, nthreads_hw>, warp_size>(
-                ctx, dst, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);
-            return;
-        }
+    // 256 threads would overflow the 64 KB work-group local memory at D == 512, so keep 128 there.
+    if (D <= 256 && nthreads == 256) {
+        constexpr int nthreads_hw = 256;
+        constexpr int nwarps = nthreads_hw / warp_size;
+        launch_fattn<D, cols_per_block, 1,
+                     flash_attn_ext_vec<D, cols_per_block, type_K, type_V,
+                                        use_logit_softcap, warp_size, nthreads_hw>, warp_size>(
+            ctx, dst, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);
+    } else {
+        constexpr int nthreads_hw = 128;
+        constexpr int nwarps = nthreads_hw / warp_size;
+        launch_fattn<D, cols_per_block, 1,
+                     flash_attn_ext_vec<D, cols_per_block, type_K, type_V,
+                                        use_logit_softcap, warp_size, nthreads_hw>, warp_size>(
+            ctx, dst, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);
     }
-
-    constexpr int nthreads_hw = 128;
-    constexpr int nwarps = nthreads_hw / warp_size;
-    launch_fattn<D, cols_per_block, 1,
-                 flash_attn_ext_vec<D, cols_per_block, type_K, type_V,
-                                    use_logit_softcap, warp_size, nthreads_hw>, warp_size>(
-        ctx, dst, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);
 }
 
 template <int D, int type_K, int type_V>

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -62,8 +62,6 @@
 #include "ggml-sycl/repeat_back.hpp"
 #include "ggml-sycl/set_rows.hpp"
 #include "ggml-sycl/set.hpp"
-#include "ggml-sycl/dsv4-hc.hpp"
-#include "ggml-sycl/lightning-indexer.hpp"
 #include "ggml-sycl/conv2d.hpp"
 #include "ggml-sycl/conv2d-dw.hpp"
 #include "ggml-sycl/conv2d-transpose.hpp"
@@ -4944,18 +4942,6 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg
         case GGML_OP_SET_ROWS:
             ggml_sycl_op_set_rows(ctx, dst);
             break;
-        case GGML_OP_DSV4_HC_PRE:
-            ggml_sycl_op_dsv4_hc_pre(ctx, dst);
-            break;
-        case GGML_OP_DSV4_HC_COMB:
-            ggml_sycl_op_dsv4_hc_comb(ctx, dst);
-            break;
-        case GGML_OP_DSV4_HC_POST:
-            ggml_sycl_op_dsv4_hc_post(ctx, dst);
-            break;
-        case GGML_OP_LIGHTNING_INDEXER:
-            ggml_sycl_op_lightning_indexer(ctx, dst);
-            break;
         case GGML_OP_DUP:
             ggml_sycl_dup(ctx, dst);
             break;
@@ -5809,33 +5795,17 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
 
         case GGML_OP_SET_ROWS:
             {
-                auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 ||
-                            op->src[0]->type == GGML_TYPE_BF16) &&
-                           (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32);
+
+                auto res = ((op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 ||
+                         op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q5_0 ||
+                         op->type == GGML_TYPE_Q1_0 ||
+                         op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_IQ4_NL ||
+                         op->type == GGML_TYPE_MXFP4 || op->type == GGML_TYPE_NVFP4) &&
+                        op->src[0]->type == GGML_TYPE_F32 &&
+                        (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32));
                 return res;
             }
             break;
-        case GGML_OP_DSV4_HC_PRE:
-            return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 &&
-                op->type == GGML_TYPE_F32;
-        case GGML_OP_DSV4_HC_COMB:
-            return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 &&
-                op->src[2]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
-        case GGML_OP_DSV4_HC_POST:
-            return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 &&
-                op->src[2]->type == GGML_TYPE_F32 && op->src[3]->type == GGML_TYPE_F32 &&
-                op->type == GGML_TYPE_F32;
-        case GGML_OP_LIGHTNING_INDEXER:
-            return op->src[0]->type == GGML_TYPE_F32 &&
-                (op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32 ||
-                 op->src[1]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_Q8_0 ||
-                 op->src[1]->type == GGML_TYPE_Q5_1 || op->src[1]->type == GGML_TYPE_Q5_0 ||
-                 op->src[1]->type == GGML_TYPE_Q4_1 || op->src[1]->type == GGML_TYPE_Q4_0 ||
-                 op->src[1]->type == GGML_TYPE_IQ4_NL) &&
-                op->src[2]->type == GGML_TYPE_F32 &&
-                op->src[3]->type == GGML_TYPE_F16 &&
-                op->type == GGML_TYPE_F32 &&
-                op->src[0]->ne[0] == WARP_SIZE * 8;
         case GGML_OP_CPY:
             {
                 ggml_type src0_type = op->src[0]->type;

ggml/src/ggml-sycl/lightning-indexer.cpp

@@ -1,197 +0,0 @@
-#include "lightning-indexer.hpp"
-#include "dequantize.hpp"
-
-static void lightning_indexer_f32_sycl(
-        const char * q, const char * k, const char * w, const char * m, float * dst,
-        int64_t n_embd, int64_t n_head, int64_t n_batch, int64_t n_stream, int64_t n_kv,
-        int64_t nem3,
-        int64_t nbq1, int64_t nbq2, int64_t nbq3,
-        int64_t nbk2, int64_t nbk3,
-        int64_t nbw1, int64_t nbw3,
-        int64_t nbm1, int64_t nbm3,
-        int64_t nb1, int64_t nb3,
-        ggml_type k_type,
-        queue_ptr stream) {
-
-    constexpr int64_t LANES = WARP_SIZE;
-    constexpr int64_t ELEMS_PER_LANE = 8;
-    constexpr int64_t ROWS_PER_BLOCK = 4;
-    constexpr int64_t BLOCK_SIZE = ROWS_PER_BLOCK * LANES;
-
-    const int64_t n_rows = n_batch * n_stream * n_kv;
-    const int64_t n_blocks = (n_rows + ROWS_PER_BLOCK - 1) / ROWS_PER_BLOCK;
-
-    stream->parallel_for(
-        sycl::nd_range<1>(
-            sycl::range<1>(n_blocks * BLOCK_SIZE),
-            sycl::range<1>(BLOCK_SIZE)),
-        [=](sycl::nd_item<1> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-            const int64_t ir   = item.get_global_id(0);
-            const int64_t lane = ir % LANES;
-            const int64_t row  = ir / LANES;
-            if (row >= n_rows) {
-                return;
-            }
-
-            const int64_t i_bs     = row / n_kv;
-            const int64_t i_kv     = row % n_kv;
-            const int64_t i_batch  = i_bs / n_stream;
-            const int64_t i_stream = i_bs % n_stream;
-
-            // load K row slice into registers (row is contiguous, nbk0 == type size)
-            const char * k_base = k + i_kv*nbk2 + i_stream*nbk3;
-            float k_local[ELEMS_PER_LANE];
-            if (k_type == GGML_TYPE_F16) {
-                const sycl::half * k_row = (const sycl::half *) k_base;
-#pragma unroll
-                for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
-                    k_local[j] = static_cast<float>(k_row[lane*ELEMS_PER_LANE + j]);
-                }
-            } else if (k_type == GGML_TYPE_F32) {
-                const float * k_row = (const float *) k_base;
-#pragma unroll
-                for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
-                    k_local[j] = k_row[lane*ELEMS_PER_LANE + j];
-                }
-            } else {
-                const int64_t lane_base = lane * ELEMS_PER_LANE;
-                switch (k_type) {
-                    case GGML_TYPE_BF16: {
-                        const sycl::ext::oneapi::bfloat16 * k_row = (const sycl::ext::oneapi::bfloat16 *) k_base;
-#pragma unroll
-                        for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
-                            k_local[j] = static_cast<float>(k_row[lane_base + j]);
-                        }
-                    } break;
-                    case GGML_TYPE_Q4_0:
-                    case GGML_TYPE_Q4_1:
-                    case GGML_TYPE_Q5_0:
-                    case GGML_TYPE_Q5_1: {
-#pragma unroll
-                        for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
-                            const int64_t idx = lane_base + j;
-                            const int64_t ib  = idx / QK4_0;
-                            const int iqs     = idx % (QK4_0/2);
-                            dfloat2 kv;
-                            if (k_type == GGML_TYPE_Q4_0) {
-                                dequantize_q4_0(k_base, ib, iqs, kv);
-                            } else if (k_type == GGML_TYPE_Q4_1) {
-                                dequantize_q4_1(k_base, ib, iqs, kv);
-                            } else if (k_type == GGML_TYPE_Q5_0) {
-                                dequantize_q5_0(k_base, ib, iqs, kv);
-                            } else {
-                                dequantize_q5_1(k_base, ib, iqs, kv);
-                            }
-                            k_local[j] = (idx % QK4_0) < (QK4_0/2) ? static_cast<float>(kv.x()) : static_cast<float>(kv.y());
-                        }
-                    } break;
-                    case GGML_TYPE_Q8_0: {
-#pragma unroll
-                        for (int64_t pair = 0; pair < ELEMS_PER_LANE / 2; ++pair) {
-                            const int64_t elem0 = lane_base + 2 * pair;
-                            dfloat2 kv;
-                            dequantize_q8_0(k_base, elem0 / QK8_0, elem0 % QK8_0, kv);
-                            k_local[2 * pair + 0] = static_cast<float>(kv.x());
-                            k_local[2 * pair + 1] = static_cast<float>(kv.y());
-                        }
-                    } break;
-                    case GGML_TYPE_IQ4_NL: {
-#pragma unroll
-                        for (int64_t pair = 0; pair < ELEMS_PER_LANE / 2; ++pair) {
-                            const int64_t elem0 = lane_base + 2 * pair;
-                            dfloat2 kv;
-                            dequantize_iq4_nl(k_base, elem0 / QK4_NL, elem0 % QK4_NL, kv);
-                            k_local[2 * pair + 0] = static_cast<float>(kv.x());
-                            k_local[2 * pair + 1] = static_cast<float>(kv.y());
-                        }
-                    } break;
-                    default:
-#pragma unroll
-                        for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
-                            k_local[j] = 0.0f;
-                        }
-                        break;
-                }
-            }
-
-            const char  * q_base = q + i_batch*nbq2 + i_stream*nbq3;
-            const float * w_base = (const float *) (w + i_batch*nbw1 + i_stream*nbw3);
-
-            float score = 0.0f;
-            for (int64_t h = 0; h < n_head; ++h) {
-                const float * q_row = (const float *) (q_base + h*nbq1);
-                float dot = 0.0f;
-#pragma unroll
-                for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
-                    const int64_t i = lane*ELEMS_PER_LANE + j;
-                    if (i < n_embd) {
-                        dot += q_row[i] * k_local[j];
-                    }
-                }
-                dot = sycl::reduce_over_group(item.get_sub_group(), dot, sycl::plus<float>());
-                if (lane == 0) {
-                    score += sycl::max(dot, 0.0f) * w_base[h];
-                }
-            }
-
-            if (lane == 0) {
-                const sycl::half * m_base = (const sycl::half *) (m + i_batch*nbm1 + (i_stream % nem3)*nbm3);
-                // flat-index store: storing through a strided base pointer
-                // hangs/misroutes writes on this stack when n_batch*n_stream > 1
-                const int64_t dst_idx = i_kv + i_batch*(nb1/sizeof(float)) + i_stream*(nb3/sizeof(float));
-                dst[dst_idx] = score + static_cast<float>(m_base[i_kv]);
-            }
-        });
-}
-
-void ggml_sycl_op_lightning_indexer(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/4);
-    const ggml_tensor * q = dst->src[0];
-    const ggml_tensor * k = dst->src[1];
-    const ggml_tensor * w = dst->src[2]; // weights
-    const ggml_tensor * m = dst->src[3]; // mask
-
-    GGML_ASSERT(dst->type == GGML_TYPE_F32);
-    GGML_ASSERT(  q->type == GGML_TYPE_F32);
-    GGML_ASSERT(  w->type == GGML_TYPE_F32);
-    GGML_ASSERT(  m->type == GGML_TYPE_F16);
-    GGML_ASSERT(k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_BF16 ||
-                k->type == GGML_TYPE_Q8_0 || k->type == GGML_TYPE_Q5_1 || k->type == GGML_TYPE_Q5_0 ||
-                k->type == GGML_TYPE_Q4_1 || k->type == GGML_TYPE_Q4_0 || k->type == GGML_TYPE_IQ4_NL);
-
-    GGML_TENSOR_LOCALS(int64_t, neq, q, ne);
-    GGML_TENSOR_LOCALS(size_t,  nbq, q, nb);
-    GGML_TENSOR_LOCALS(int64_t, nek, k, ne);
-    GGML_TENSOR_LOCALS(size_t,  nbk, k, nb);
-    GGML_TENSOR_LOCALS(size_t,  nbw, w, nb);
-    GGML_TENSOR_LOCALS(int64_t, nem, m, ne);
-    GGML_TENSOR_LOCALS(size_t,  nbm, m, nb);
-    GGML_TENSOR_LOCALS(int64_t, ne, dst, ne);
-    GGML_TENSOR_LOCALS(size_t,  nb, dst, nb);
-
-    // input rows must be contiguous
-    GGML_ASSERT(nbq0 == ggml_type_size(q->type));
-    GGML_ASSERT(nbk0 == ggml_type_size(k->type));
-    GGML_ASSERT(nbm0 == ggml_type_size(m->type));
-    GGML_ASSERT(nb0  == ggml_type_size(dst->type));
-
-    const int64_t n_embd   = neq0;
-    const int64_t n_head   = neq1;
-    const int64_t n_batch  = neq2;
-    const int64_t n_stream = neq3;
-    const int64_t n_kv     = nek2;
-
-    GGML_ASSERT(n_embd == WARP_SIZE * 8);
-
-    lightning_indexer_f32_sycl(
-            (const char *) q->data, (const char *) k->data,
-            (const char *) w->data, (const char *) m->data, (float *) dst->data,
-            n_embd, n_head, n_batch, n_stream, n_kv, nem3,
-            nbq1, nbq2, nbq3,
-            nbk2, nbk3,
-            nbw1, nbw3,
-            nbm1, nbm3,
-            nb1, nb3,
-            k->type,
-            ctx.stream());
-}

ggml/src/ggml-sycl/lightning-indexer.hpp

@@ -1,8 +0,0 @@
-#ifndef GGML_SYCL_LIGHTNING_INDEXER_HPP
-#define GGML_SYCL_LIGHTNING_INDEXER_HPP
-
-#include "common.hpp"
-
-void ggml_sycl_op_lightning_indexer(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
-
-#endif // GGML_SYCL_LIGHTNING_INDEXER_HPP

ggml/src/ggml-sycl/presets.hpp

@@ -20,6 +20,8 @@
 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses
 
 #define SYCL_COL2IM_1D_BLOCK_SIZE 256
+#define SYCL_GELU_BLOCK_SIZE 256
+#define SYCL_SILU_BLOCK_SIZE 256
 #define SYCL_TANH_BLOCK_SIZE 256
 #define SYCL_RELU_BLOCK_SIZE 256
 #define SYCL_HARDSIGMOID_BLOCK_SIZE 256

ggml/src/ggml-sycl/set_rows.cpp

@@ -1,10 +1,6 @@
 #include "set_rows.hpp"
 #include "cpy.hpp"
 
-#include "ggml-quants.h"
-
-#include <vector>
-
 namespace utils {
 template<typename T>
 static constexpr bool is_arithmetic_v() {
@@ -24,17 +20,7 @@ convert (const char* src, char* dst) {
    *reinterpret_cast<TOut*>(dst) = dst_val;
 }
 
-#ifdef GGML_SYCL_HAS_BF16
-// sycl::vec::convert does not provide a half -> bfloat16 path, so route through float.
-template<>
-inline void convert<sycl::half, sycl::ext::oneapi::bfloat16>(const char* src, char* dst) {
-    const float tmp = sycl::vec<sycl::half, 1>(*reinterpret_cast<const sycl::half*>(src))
-                          .template convert<float, sycl::rounding_mode::automatic>()[0];
-    *reinterpret_cast<sycl::ext::oneapi::bfloat16*>(dst) = sycl::ext::oneapi::bfloat16(tmp);
-}
-#endif
-
-template <typename TIn, typename TIdx, typename blockType, int qk, cpy_kernel_t cpyblck>
+template <typename TIdx, typename blockType, int qk, cpy_kernel_t cpyblck>
 static void set_rows_sycl_q(const char * __restrict__ src0_d,
                             const TIdx * __restrict__ src1_d,
                             blockType * __restrict__ dst_d,
@@ -82,22 +68,13 @@ static void set_rows_sycl_q(const char * __restrict__ src0_d,
         const int64_t i11         = i02 % ne11;
         const int64_t i10         = i01;
         const size_t  src_offset  = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
-        const char *  src_block   = src0_d + src_offset + i00 * sizeof(TIn);
+        const char *  src_block   = src0_d + src_offset + i00 * sizeof(float);
         const size_t  src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
         const int64_t dst_row     = src1_d[src1_offset / sizeof(TIdx)];
         const size_t  dst_offset =
             calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 }) + (i00 / qk) * sizeof(blockType);
         char * dst_block = reinterpret_cast<char *>(reinterpret_cast<char *>(dst_d) + dst_offset);
-        if constexpr (std::is_same_v<TIn, float>) {
-            cpyblck(src_block, dst_block);
-        } else {
-            float src_block_f32[qk];
-            const TIn * src_block_t = reinterpret_cast<const TIn *>(src_block);
-            for (int j = 0; j < qk; ++j) {
-                src_block_f32[j] = (float) src_block_t[j];
-            }
-            cpyblck(reinterpret_cast<const char *>(src_block_f32), dst_block);
-        }
+        cpyblck(src_block, dst_block);
     });
     GGML_UNUSED(ne10);
     GGML_UNUSED(ne13);
@@ -105,139 +82,6 @@ static void set_rows_sycl_q(const char * __restrict__ src0_d,
     GGML_UNUSED(nb13);
 }
 
-template<typename blockType>
-using quantize_row_qk_t = void (*)(const float *, blockType *, int64_t);
-
-using quantize_rows_f_t = size_t (*)(const float *, void *, int64_t, int64_t, const float *);
-
-template <typename TIn, typename TIdx, typename blockType, int qk, quantize_row_qk_t<blockType> quantize_row>
-static void set_rows_sycl_qk_host(
-        const ggml_tensor * src0,
-        const ggml_tensor * src1,
-        ggml_tensor * dst,
-        const int64_t ne00,
-        const int64_t ne01,
-        const int64_t ne02,
-        const int64_t ne03,
-        const int64_t ne11,
-        const int64_t ne12,
-        const size_t nb01,
-        const size_t nb02,
-        const size_t nb03,
-        const size_t nb10,
-        const size_t nb11,
-        const size_t nb12,
-        const size_t nb1,
-        const size_t nb2,
-        const size_t nb3,
-        queue_ptr stream) {
-    GGML_ASSERT(ne00 % qk == 0);
-
-    const size_t src0_bytes = ggml_nbytes(src0);
-    const size_t src1_bytes = ggml_nbytes(src1);
-
-    std::vector<char> src0_host(src0_bytes);
-    std::vector<char> src1_host(src1_bytes);
-
-    stream->memcpy(src0_host.data(), src0->data, src0_bytes);
-    stream->memcpy(src1_host.data(), src1->data, src1_bytes);
-    stream->wait();
-
-    std::vector<float> src_row_f32(ne00);
-    const int64_t nblocks = ne00 / qk;
-    std::vector<blockType> dst_row_q(nblocks);
-
-    for (int64_t i03 = 0; i03 < ne03; ++i03) {
-        for (int64_t i02 = 0; i02 < ne02; ++i02) {
-            for (int64_t i01 = 0; i01 < ne01; ++i01) {
-                const int64_t i12 = i03 % ne12;
-                const int64_t i11 = i02 % ne11;
-                const int64_t i10 = i01;
-
-                const size_t src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
-                const int64_t dst_row = *(const TIdx *) (src1_host.data() + src1_offset);
-
-                const size_t src0_row_offset = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
-                const TIn * src_row = reinterpret_cast<const TIn *>(src0_host.data() + src0_row_offset);
-
-                for (int64_t i00 = 0; i00 < ne00; ++i00) {
-                    src_row_f32[i00] = (float) src_row[i00];
-                }
-
-                quantize_row(src_row_f32.data(), dst_row_q.data(), ne00);
-
-                const size_t dst_offset = calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 });
-                stream->memcpy((char *) dst->data + dst_offset, dst_row_q.data(), nblocks * sizeof(blockType));
-                stream->wait();
-            }
-        }
-    }
-}
-
-template <typename TIn, typename TIdx, typename blockType, int qk, quantize_rows_f_t quantize_rows>
-static void set_rows_sycl_iq_host(
-        const ggml_tensor * src0,
-        const ggml_tensor * src1,
-        ggml_tensor * dst,
-        const int64_t ne00,
-        const int64_t ne01,
-        const int64_t ne02,
-        const int64_t ne03,
-        const int64_t ne11,
-        const int64_t ne12,
-        const size_t nb01,
-        const size_t nb02,
-        const size_t nb03,
-        const size_t nb10,
-        const size_t nb11,
-        const size_t nb12,
-        const size_t nb1,
-        const size_t nb2,
-        const size_t nb3,
-        queue_ptr stream) {
-    GGML_ASSERT(ne00 % qk == 0);
-
-    const size_t src0_bytes = ggml_nbytes(src0);
-    const size_t src1_bytes = ggml_nbytes(src1);
-
-    std::vector<char> src0_host(src0_bytes);
-    std::vector<char> src1_host(src1_bytes);
-
-    stream->memcpy(src0_host.data(), src0->data, src0_bytes);
-    stream->memcpy(src1_host.data(), src1->data, src1_bytes);
-    stream->wait();
-
-    std::vector<float> src_row_f32(ne00);
-    const int64_t nblocks = ne00 / qk;
-    std::vector<blockType> dst_row_q(nblocks);
-
-    for (int64_t i03 = 0; i03 < ne03; ++i03) {
-        for (int64_t i02 = 0; i02 < ne02; ++i02) {
-            for (int64_t i01 = 0; i01 < ne01; ++i01) {
-                const int64_t i12 = i03 % ne12;
-                const int64_t i11 = i02 % ne11;
-                const int64_t i10 = i01;
-
-                const size_t src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
-                const int64_t dst_row = *(const TIdx *) (src1_host.data() + src1_offset);
-
-                const size_t src0_row_offset = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
-                const TIn * src_row = reinterpret_cast<const TIn *>(src0_host.data() + src0_row_offset);
-
-                for (int64_t i00 = 0; i00 < ne00; ++i00) {
-                    src_row_f32[i00] = (float) src_row[i00];
-                }
-
-                quantize_rows(src_row_f32.data(), dst_row_q.data(), 1, ne00, nullptr);
-
-                const size_t dst_offset = calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 });
-                stream->memcpy((char *) dst->data + dst_offset, dst_row_q.data(), nblocks * sizeof(blockType));
-                stream->wait();
-            }
-        }
-    }
-}
-
 template<typename TIn, typename TIdx, typename TOut>
 static void k_set_rows(
         const char * __restrict__ src0, const TIdx * __restrict__ src1, char * __restrict__ dst,
@@ -356,194 +200,31 @@ static void set_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor * s
             break;
 #endif
         case GGML_TYPE_Q8_0:
-            set_rows_sycl_q<TIn, TIdx, block_q8_0, QK8_0, cpy_blck_f32_q8_0>(
-                src0_d, src1_d, (block_q8_0 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
+            set_rows_sycl_q<TIdx, block_q8_0, QK8_0, cpy_blck_f32_q8_0>(src0_d, src1_d, (block_q8_0 *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         case GGML_TYPE_Q1_0:
-            set_rows_sycl_q<TIn, TIdx, block_q1_0, QK1_0, cpy_blck_f32_q1_0>(
-                src0_d, src1_d, (block_q1_0 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
-            break;
-        case GGML_TYPE_Q2_0:
-            set_rows_sycl_q<TIn, TIdx, block_q2_0, QK2_0, cpy_blck_f32_q2_0>(
-                src0_d, src1_d, (block_q2_0 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
+            set_rows_sycl_q<TIdx, block_q1_0, QK1_0, cpy_blck_f32_q1_0>(src0_d, src1_d, (block_q1_0 *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         case GGML_TYPE_Q5_1:
-            set_rows_sycl_q<TIn, TIdx, block_q5_1, QK5_1, cpy_blck_f32_q5_1>(
-                src0_d, src1_d, (block_q5_1 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
+            set_rows_sycl_q<TIdx, block_q5_1, QK5_1, cpy_blck_f32_q5_1>(src0_d, src1_d, (block_q5_1 *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         case GGML_TYPE_Q5_0:
-            set_rows_sycl_q<TIn, TIdx, block_q5_0, QK5_0, cpy_blck_f32_q5_0>(
-                src0_d, src1_d, (block_q5_0 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
+            set_rows_sycl_q<TIdx, block_q5_0, QK5_0, cpy_blck_f32_q5_0>(src0_d, src1_d, (block_q5_0 *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         case GGML_TYPE_Q4_1:
-            set_rows_sycl_q<TIn, TIdx, block_q4_1, QK4_1, cpy_blck_f32_q4_1>(
-                src0_d, src1_d, (block_q4_1 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
+            set_rows_sycl_q<TIdx, block_q4_1, QK4_1, cpy_blck_f32_q4_1>(src0_d, src1_d, (block_q4_1 *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         case GGML_TYPE_Q4_0:
-            set_rows_sycl_q<TIn, TIdx, block_q4_0, QK4_0, cpy_blck_f32_q4_0>(
-                src0_d, src1_d, (block_q4_0 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
+            set_rows_sycl_q<TIdx, block_q4_0, QK4_0, cpy_blck_f32_q4_0>(src0_d, src1_d, (block_q4_0 *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         case GGML_TYPE_IQ4_NL:
-            set_rows_sycl_q<TIn, TIdx, block_iq4_nl, QK4_NL, cpy_blck_f32_iq4_nl>(
-                src0_d, src1_d, (block_iq4_nl *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
+            set_rows_sycl_q<TIdx, block_iq4_nl, QK4_NL, cpy_blck_f32_iq4_nl>(src0_d, src1_d, (block_iq4_nl *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         case GGML_TYPE_MXFP4:
-            set_rows_sycl_q<TIn, TIdx, block_mxfp4, QK_MXFP4, cpy_blck_f32_mxfp4>(
-                src0_d, src1_d, (block_mxfp4 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
+            set_rows_sycl_q<TIdx, block_mxfp4, QK_MXFP4, cpy_blck_f32_mxfp4>(src0_d, src1_d, (block_mxfp4 *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         case GGML_TYPE_NVFP4:
-            set_rows_sycl_q<TIn, TIdx, block_nvfp4, QK_NVFP4, cpy_blck_f32_nvfp4>(
-                src0_d, src1_d, (block_nvfp4 *) dst->data, ne00, ne01, ne02, ne03,
-                ne10, ne11, ne12, ne13, nb00, nb01,
-                nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
-            break;
-        case GGML_TYPE_Q2_K:
-            set_rows_sycl_qk_host<TIn, TIdx, block_q2_K, QK_K, quantize_row_q2_K_ref>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_Q3_K:
-            set_rows_sycl_qk_host<TIn, TIdx, block_q3_K, QK_K, quantize_row_q3_K_ref>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_Q4_K:
-            set_rows_sycl_qk_host<TIn, TIdx, block_q4_K, QK_K, quantize_row_q4_K_ref>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_Q5_K:
-            set_rows_sycl_qk_host<TIn, TIdx, block_q5_K, QK_K, quantize_row_q5_K_ref>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_Q6_K:
-            set_rows_sycl_qk_host<TIn, TIdx, block_q6_K, QK_K, quantize_row_q6_K_ref>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_IQ2_XXS:
-            set_rows_sycl_iq_host<TIn, TIdx, block_iq2_xxs, QK_K, quantize_iq2_xxs>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_IQ2_XS:
-            set_rows_sycl_iq_host<TIn, TIdx, block_iq2_xs, QK_K, quantize_iq2_xs>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_IQ2_S:
-            set_rows_sycl_iq_host<TIn, TIdx, block_iq2_s, QK_K, quantize_iq2_s>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_IQ3_XXS:
-            set_rows_sycl_qk_host<TIn, TIdx, block_iq3_xxs, QK_K, quantize_row_iq3_xxs_ref>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_IQ3_S:
-            set_rows_sycl_qk_host<TIn, TIdx, block_iq3_s, QK_K, quantize_row_iq3_s_ref>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_IQ1_S:
-            set_rows_sycl_iq_host<TIn, TIdx, block_iq1_s, QK_K, quantize_iq1_s>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_IQ1_M:
-            set_rows_sycl_iq_host<TIn, TIdx, block_iq1_m, QK_K, quantize_iq1_m>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
-            break;
-        case GGML_TYPE_IQ4_XS:
-            set_rows_sycl_qk_host<TIn, TIdx, block_iq4_xs, QK_K, quantize_row_iq4_xs_ref>(
-                src0, src1, dst,
-                ne00, ne01, ne02, ne03,
-                ne11, ne12,
-                nb01, nb02, nb03,
-                nb10, nb11, nb12,
-                nb1, nb2, nb3,
-                stream);
+            set_rows_sycl_q<TIdx, block_nvfp4, QK_NVFP4, cpy_blck_f32_nvfp4>(src0_d, src1_d, (block_nvfp4 *)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
             break;
         default:
             GGML_ABORT("Unsupported tensor type!");
@@ -556,21 +237,12 @@ void ggml_sycl_op_set_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     const ggml_tensor * src0 = dst->src[0];
     const ggml_tensor * src1 = dst->src[1];
 
-    GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
+    GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
     GGML_ASSERT(dst->src[1]->type == GGML_TYPE_I64 || dst->src[1]->type == GGML_TYPE_I32);
 
-    // dispatch on the index type (src1) and the source value type (src0)
-    if (src0->type == GGML_TYPE_F16) {
-        if (src1->type == GGML_TYPE_I64) {
-            set_rows_sycl<sycl::half, int64_t>(ctx, src0, src1, dst);
-        } else {
-            set_rows_sycl<sycl::half, int32_t>(ctx, src0, src1, dst);
-        }
+    if (src1->type == GGML_TYPE_I64) {
+        set_rows_sycl<float, int64_t>(ctx, src0, src1, dst);
     } else {
-        if (src1->type == GGML_TYPE_I64) {
-            set_rows_sycl<float, int64_t>(ctx, src0, src1, dst);
-        } else {
-            set_rows_sycl<float, int32_t>(ctx, src0, src1, dst);
-        }
+        set_rows_sycl<float, int32_t>(ctx, src0, src1, dst);
     }
 }

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -186,22 +186,13 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
 
 #define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256
 
-#define VK_CHECK(err, msg, dev)                                     \
+#define VK_CHECK(err, msg)                                          \
     do {                                                            \
-        vk::Result err_;                                            \
-        try {                                                       \
-            err_ = (err);                                           \
-        } catch (vk::DeviceLostError &) {                           \
-            ggml_vk_print_device_lost_info(dev);                    \
-            GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n",            \
-                #err, __FILE__, __LINE__);                          \
-            throw;                                                  \
-        }                                                           \
+        vk::Result err_ = (err);                                    \
         if (err_ != vk::Result::eSuccess) {                         \
-            GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n",   \
+            fprintf(stderr, "ggml_vulkan: %s error %s at %s:%d\n",  \
                 #err, to_string(err_).c_str(), __FILE__, __LINE__); \
-            throw vk::SystemError(vk::make_error_code(err_),        \
-                "ggml_vulkan: " msg);                               \
+            exit(1);                                                \
         }                                                           \
     } while (0)
 
@@ -311,13 +302,9 @@ struct vk_command_pool {
     }
 };
 
-static void ggml_vk_print_device_fault_info(const vk_device& device);
-static void ggml_vk_print_device_lost_info(const vk_device& device);
-
 // Prevent simultaneous submissions to the same queue.
 struct vk_queue_handle {
     vk::Queue queue;
-    vk_device_ref device;
     virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0;
     virtual void lock()   {}   // no-op by default (internally synchronized case)
     virtual void unlock() {}
@@ -328,14 +315,7 @@ struct vk_queue_handle_synchronized : vk_queue_handle {
     std::mutex mutex;
     void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {
         std::lock_guard<std::mutex> guard(mutex);
-        try {
-            queue.submit(submits, fence);
-        } catch (vk::DeviceLostError &) {
-            if (auto dev = device.lock()) {
-                ggml_vk_print_device_lost_info(dev);
-            }
-            throw;
-        }
+        queue.submit(submits, fence);
     }
     void lock()   override { mutex.lock(); }
     void unlock() override { mutex.unlock(); }
@@ -344,14 +324,7 @@ struct vk_queue_handle_synchronized : vk_queue_handle {
 struct vk_queue_handle_unsynchronized : vk_queue_handle {
     void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {
         // Driver guarantees internal synchronization via VK_KHR_internally_synchronized_queues
-        try {
-            queue.submit(submits, fence);
-        } catch (vk::DeviceLostError &) {
-            if (auto dev = device.lock()) {
-                ggml_vk_print_device_lost_info(dev);
-            }
-            throw;
-        }
+        queue.submit(submits, fence);
     }
     // lock()/unlock() inherited no-ops
 };
@@ -862,15 +835,6 @@ struct vk_device_struct {
 
     bool pipeline_executable_properties_support {};
 
-    bool device_fault {};
-    PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {};
-
-    bool serialize_submissions {};
-
-    const ggml_cgraph * diag_cgraph {};
-    int diag_prev_start = -1;
-    int diag_prev_end = -1;
-
     size_t idx;
 
     bool mul_mat_l[GGML_TYPE_COUNT];
@@ -1154,57 +1118,6 @@ void vk_command_pool::destroy(vk::Device& device) {
     cmd_buffers.clear();
 }
 
-static void ggml_vk_print_device_fault_info(const vk_device& device) {
-    if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) {
-        return;
-    }
-
-    VkDeviceFaultCountsEXT fault_counts {};
-    fault_counts.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT;
-    VkResult res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, nullptr);
-    if (res != VK_SUCCESS) {
-        GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (counts) failed: %d\n", res);
-        return;
-    }
-
-    std::vector<VkDeviceFaultAddressInfoEXT> address_infos(fault_counts.addressInfoCount);
-    std::vector<VkDeviceFaultVendorInfoEXT> vendor_infos(fault_counts.vendorInfoCount);
-
-    VkDeviceFaultInfoEXT fault_info {};
-    fault_info.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT;
-    fault_info.pAddressInfos = address_infos.data();
-    fault_info.pVendorInfos = vendor_infos.data();
-
-    res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, &fault_info);
-    if (res != VK_SUCCESS) {
-        GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (info) failed: %d\n", res);
-        return;
-    }
-
-    if (fault_counts.addressInfoCount == 0 && fault_counts.vendorInfoCount == 0 && fault_info.description[0] == '\0') {
-        return;
-    }
-
-    if (fault_info.description[0] != '\0') {
-        GGML_LOG_ERROR("ggml_vulkan: device fault on %s: %s\n", device->name.c_str(), fault_info.description);
-    }
-
-    for (uint32_t i = 0; i < fault_counts.addressInfoCount; i++) {
-        const auto& info = address_infos[i];
-        GGML_LOG_CONT("  address fault %u: type=%d address=0x%llx precision=0x%llx\n",
-                i, (int)info.addressType,
-                (unsigned long long)info.reportedAddress,
-                (unsigned long long)info.addressPrecision);
-    }
-    for (uint32_t i = 0; i < fault_counts.vendorInfoCount; i++) {
-        const auto& info = vendor_infos[i];
-        GGML_LOG_CONT("  vendor fault %u: %s (code=0x%llx data=0x%llx)\n",
-                i, info.description,
-                (unsigned long long)info.vendorFaultCode,
-                (unsigned long long)info.vendorFaultData);
-    }
-}
-
 struct vk_buffer_struct {
     vk::Buffer buffer = VK_NULL_HANDLE;
     vk::DeviceMemory device_memory = VK_NULL_HANDLE;
@@ -2146,36 +2059,6 @@ static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {
     return 0;
 }
 
-static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) {
-    uint64_t total_flops = 0;
-    int n_ops = 0;
-    for (int j = start; j <= end && j < cgraph->n_nodes; j++) {
-        uint64_t flops = ggml_vk_get_node_flops(cgraph->nodes[j]);
-        total_flops += flops;
-        n_ops++;
-        if (flops > 0) {
-            GGML_LOG_CONT("  node %d: %s (%s) [%.2f GFLOP]\n",
-                    j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op),
-                    flops / 1e9);
-        } else {
-            GGML_LOG_CONT("  node %d: %s (%s)\n",
-                    j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op));
-        }
-    }
-    GGML_LOG_CONT("  total: %d ops, %.2f GFLOP\n", n_ops, total_flops / 1e9);
-}
-
-static void ggml_vk_print_device_lost_info(const vk_device& device) {
-    ggml_vk_print_device_fault_info(device);
-    if (device->serialize_submissions && device->diag_cgraph != nullptr && device->diag_prev_start >= 0) {
-        GGML_LOG_ERROR("ggml_vulkan: device lost on %s, likely caused by previous submission (nodes %d to %d):\n",
-                device->name.c_str(), device->diag_prev_start, device->diag_prev_end);
-        ggml_vk_print_node_list(device->diag_cgraph, device->diag_prev_start, device->diag_prev_end);
-    } else {
-        GGML_LOG_ERROR("ggml_vulkan: device lost on %s\n", device->name.c_str());
-    }
-}
-
 class vk_perf_logger {
   public:
     void print_timings(bool force = false) {
@@ -2588,27 +2471,17 @@ static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {
     // Use waitForFences while most of the graph executes. Hopefully the CPU can sleep
     // during this wait.
     if (ctx->almost_ready_fence_pending) {
-        VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence", ctx->device);
+        VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence");
         ctx->device->device.resetFences({ ctx->almost_ready_fence });
         ctx->almost_ready_fence_pending = false;
     }
 
     // Spin (w/pause) waiting for the graph to finish executing.
     vk::Result result;
-    for (;;) {
-        try {
-            result = ctx->device->device.getFenceStatus(ctx->fence);
-        } catch (vk::DeviceLostError &) {
-            ggml_vk_print_device_lost_info(ctx->device);
-            GGML_LOG_ERROR("ggml_vulkan: getFenceStatus at %s:%d\n", __FILE__, __LINE__);
-            throw;
-        }
-        if (result == vk::Result::eSuccess) {
-            break;
-        }
+    while ((result = ctx->device->device.getFenceStatus(ctx->fence)) != vk::Result::eSuccess) {
         if (result != vk::Result::eNotReady) {
-            GGML_LOG_ERROR("ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__);
-            throw vk::SystemError(vk::make_error_code(result), "ggml_vulkan: getFenceStatus");
+            fprintf(stderr, "ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__);
+            exit(1);
         }
         for (uint32_t i = 0; i < 100; ++i) {
             YIELD();
@@ -3299,7 +3172,6 @@ static std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_
     }
 
     h->queue = device->device.getQueue2(queue_info2);
-    h->device = device;
     q->handle = h;
 
     q->cmd_pool.init(device, q.get());
@@ -6245,8 +6117,6 @@ static vk_device ggml_vk_get_device(size_t idx) {
 #endif
             } else if (strcmp(VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME, properties.extensionName) == 0) {
                 internally_sync_support = true;
-            } else if (strcmp("VK_EXT_device_fault", properties.extensionName) == 0) {
-                device->device_fault = true;
             }
         }
 
@@ -6601,18 +6471,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
         }
 #endif
 
-        VkPhysicalDeviceFaultFeaturesEXT fault_features {};
-        fault_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT;
-        if (device->device_fault) {
-            last_struct->pNext = (VkBaseOutStructure *)&fault_features;
-            last_struct = (VkBaseOutStructure *)&fault_features;
-            device_extensions.push_back("VK_EXT_device_fault");
-        }
-
         vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2);
 
-        device->device_fault = device->device_fault && fault_features.deviceFault;
-
         device->has_internally_synchronized_queues = internally_synchronized_queues_features.internallySynchronizedQueues;
 
         // Build queue create infos only after querying whether internally synchronized queues are enabled.
@@ -6911,11 +6771,6 @@ static vk_device ggml_vk_get_device(size_t idx) {
         device_create_info.setPNext(&device_features2);
         device->device = device->physical_device.createDevice(device_create_info);
 
-        if (device->device_fault) {
-            device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT)
-                vkGetDeviceProcAddr(device->device, "vkGetDeviceFaultInfoEXT");
-        }
-
         // Queues
         device->compute_queue = ggml_vk_create_queue(device, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }, false);
 
@@ -7038,8 +6893,6 @@ static vk_device ggml_vk_get_device(size_t idx) {
 
         device->idx = idx;
 
-        device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;
-
         device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr;
 
         device->add_rms_fusion = !device->disable_fusion &&
@@ -8466,7 +8319,7 @@ static void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void *
         }
 
         ggml_vk_submit(subctx, dst->device->fence);
-        VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);
+        VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences");
         dst->device->device.resetFences({ dst->device->fence });
         ggml_vk_queue_command_pools_cleanup(dst->device);
     }
@@ -8578,7 +8431,7 @@ static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, si
         ggml_vk_ctx_end(subctx);
         ggml_vk_submit(subctx, src->device->fence);
         VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),
-                 "vk_buffer_read_2d uma waitForFences", src->device);
+                 "vk_buffer_read_2d uma waitForFences");
         src->device->device.resetFences({ src->device->fence });
         ggml_vk_queue_command_pools_cleanup(src->device);
 
@@ -8599,7 +8452,7 @@ static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, si
         ggml_vk_ctx_end(subctx);
 
         ggml_vk_submit(subctx, src->device->fence);
-        VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);
+        VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences");
         src->device->device.resetFences({ src->device->fence });
         ggml_vk_queue_command_pools_cleanup(src->device);
 
@@ -8634,7 +8487,7 @@ static void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& sr
         ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);
         ggml_vk_ctx_end(subctx);
         ggml_vk_submit(subctx, src->device->fence);
-        VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);
+        VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences");
         src->device->device.resetFences({ src->device->fence });
         ggml_vk_queue_command_pools_cleanup(src->device);
     } else {
@@ -8678,7 +8531,7 @@ static void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, siz
     ggml_vk_ctx_end(subctx);
 
     ggml_vk_submit(subctx, dst->device->fence);
-    VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);
+    VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences");
     dst->device->device.resetFences({ dst->device->fence });
     ggml_vk_queue_command_pools_cleanup(dst->device);
 }
@@ -14413,7 +14266,7 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
 
     auto begin = std::chrono::high_resolution_clock::now();
     ggml_vk_submit(subctx, ctx->fence);
-    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device);
+    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences");
     ctx->device->device.resetFences({ ctx->fence });
     ggml_vk_queue_command_pools_cleanup(ctx->device);
 
@@ -14615,7 +14468,7 @@ static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_
     auto begin = std::chrono::high_resolution_clock::now();
 
     ggml_vk_submit(subctx, ctx->fence);
-    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
+    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences");
     ctx->device->device.resetFences({ ctx->fence });
     ggml_vk_queue_command_pools_cleanup(ctx->device);
 
@@ -14901,7 +14754,7 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m,
     auto begin = std::chrono::high_resolution_clock::now();
 
     ggml_vk_submit(subctx, ctx->fence);
-    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
+    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences");
     ctx->device->device.resetFences({ ctx->fence });
     ggml_vk_queue_command_pools_cleanup(ctx->device);
 
@@ -15700,9 +15553,7 @@ static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph *
             memset(mset.dst, mset.val, mset.n);
         }
 
-        if (ctx->device->serialize_submissions) {
-            ggml_vk_submit(subctx, ctx->fence);
-        } else if (almost_ready && !ctx->almost_ready_fence_pending) {
+        if (almost_ready && !ctx->almost_ready_fence_pending) {
             ggml_vk_submit(subctx, ctx->almost_ready_fence);
             ctx->almost_ready_fence_pending = true;
         } else {
@@ -16313,20 +16164,12 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
             memcpy(cpy.dst, cpy.src, cpy.n);
         }
 
-        if (ctx->device->serialize_submissions) {
-            ggml_vk_submit(compute_ctx, ctx->fence);
-            VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "synchronize waitForFences", ctx->device);
-            ctx->device->device.resetFences({ ctx->fence });
-        } else {
-            ggml_vk_submit(compute_ctx, {});
-        }
+        ggml_vk_submit(compute_ctx, {});
         ctx->submit_pending = true;
     }
 
     if (ctx->submit_pending) {
-        if (ctx->device->serialize_submissions) {
-            ctx->submit_pending = false;
-        } else if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) {
+        if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) {
             vk::TimelineSemaphoreSubmitInfo tl_info{
                 1, &ctx->transfer_semaphore.value,
                 0, nullptr,
@@ -16343,9 +16186,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
         } else {
             ctx->device->compute_queue->handle->submit({}, ctx->fence);
         }
-        if (!ctx->device->serialize_submissions) {
-            ggml_vk_wait_for_fence(ctx);
-        }
+        ggml_vk_wait_for_fence(ctx);
         ctx->submit_pending = false;
         if (cmd_buf) {
             cmd_buf->in_use = false;
@@ -16917,10 +16758,6 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
     VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");
     ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
 
-    ctx->device->diag_cgraph = nullptr;
-    ctx->device->diag_prev_start = -1;
-    ctx->device->diag_prev_end = -1;
-
     if (vk_instance.debug_utils_support) {
         vk::DebugUtilsLabelEXT dul = {};
         dul.pLabelName = "ggml_backend_vk_graph_compute";
@@ -17012,36 +16849,6 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
     }
     uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u);
 
-    auto const submit_after = [&](int start, int end) {
-        if (ctx->device->serialize_submissions) {
-            try {
-                auto res = ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX);
-                if (res != vk::Result::eSuccess) {
-                    GGML_LOG_ERROR("ggml_vulkan: waitForFences error during serialized submission\n");
-                    throw vk::SystemError(vk::make_error_code(res), "ggml_vulkan: waitForFences during serialized submission");
-                }
-            } catch (vk::DeviceLostError &) {
-                ggml_vk_print_device_fault_info(ctx->device);
-                GGML_LOG_ERROR("ggml_vulkan: device lost on %s waiting for submission (nodes %d to %d):\n",
-                        ctx->device->name.c_str(), start, end);
-                ggml_vk_print_node_list(cgraph, start, end);
-                throw;
-            }
-            ctx->device->device.resetFences({ ctx->fence });
-            ctx->submit_pending = false;
-            ctx->device->diag_cgraph = cgraph;
-            ctx->device->diag_prev_start = start;
-            ctx->device->diag_prev_end = end;
-        }
-        first_node_in_batch = true;
-        submitted_nodes = 0;
-        batch_flops = 0;
-        if (submit_count < 3) {
-            flops_per_submit *= 2;
-        }
-        submit_count++;
-    };
-
     for (int i = 0; i < cgraph->n_nodes; i++) {
         if (first_node_in_batch) {
             submit_node_idx = i;
@@ -17049,20 +16856,8 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
 
         {
             auto node_flops = ggml_vk_get_node_flops(cgraph->nodes[i]);
-            total_flops += node_flops;
-
-            // Flush the current batch before recording a node that would push it over the flop threshold
-            if (flops_per_submit != 0 && submitted_nodes > 0 && batch_flops + node_flops >= flops_per_submit) {
-                vk_context flush_ctx = ggml_vk_get_compute_ctx(ctx);
-                ggml_vk_ctx_end(flush_ctx);
-                flush_ctx->exit_tensor_idx = -1;
-                ctx->compute_ctx.reset();
-                ggml_vk_compute_forward(ctx, cgraph, cgraph->nodes[submit_node_idx], submit_node_idx, false);
-                submit_after(submit_node_idx, i - 1);
-                submit_node_idx = i;
-            }
-
             batch_flops += node_flops;
+            total_flops += node_flops;
         }
 
         // op_srcs_fused_elementwise indicates whether an op's srcs all contribute to
@@ -17316,7 +17111,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
         }
 
         if (submit && enqueued) {
-            submit_after(submit_node_idx, i + (int)ctx->num_additional_fused_ops);
+            first_node_in_batch = true;
+            submitted_nodes = 0;
+            batch_flops = 0;
+            if (submit_count < 3) {
+                flops_per_submit *= 2;
+            }
+            submit_count++;
         }
         i += ctx->num_additional_fused_ops;
         ctx->num_additional_fused_ops = 0;
@@ -17332,13 +17133,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
         ggml_vk_ctx_end(compute_ctx);
 
         ggml_vk_submit(compute_ctx, ctx->device->fence);
-        VK_CHECK(ctx->device->device.waitForFences({ ctx->device->fence }, true, UINT64_MAX), "GGML_VULKAN_PERF waitForFences", ctx->device);
+        VK_CHECK(ctx->device->device.waitForFences({ ctx->device->fence }, true, UINT64_MAX), "GGML_VULKAN_PERF waitForFences");
         ctx->device->device.resetFences({ ctx->device->fence });
         ctx->compute_ctx.reset();
 
         // Get the results and pass them to the logger
         std::vector<uint64_t> timestamps(cgraph->n_nodes + 1);
-        VK_CHECK(ctx->device->device.getQueryPoolResults(ctx->query_pool, 0, ctx->query_idx, (cgraph->n_nodes + 1)*sizeof(uint64_t), timestamps.data(), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait), "get timestamp results", ctx->device);
+        VK_CHECK(ctx->device->device.getQueryPoolResults(ctx->query_pool, 0, ctx->query_idx, (cgraph->n_nodes + 1)*sizeof(uint64_t), timestamps.data(), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait), "get timestamp results");
         if (!vk_perf_logger_concurrent) {
             // Log each op separately
             for (int i = 1; i < ctx->query_idx; i++) {
@@ -18565,7 +18366,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm
         vk::Semaphore sem = vkev->tl_semaphore.s;
         uint64_t val = vkev->tl_semaphore.value;
         vk::SemaphoreWaitInfo swi{vk::SemaphoreWaitFlags{}, sem, val};
-        VK_CHECK(device->device.waitSemaphores(swi, UINT64_MAX), "event_synchronize", device);
+        VK_CHECK(device->device.waitSemaphores(swi, UINT64_MAX), "event_synchronize");
 
         // Reset and move submitted events
         for (auto& event : vkev->events_submitted) {

ggml/src/ggml.c

@@ -7200,10 +7200,6 @@ void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor *
     ggml_build_forward_impl(cgraph, tensor, true, true);
 }
 
-void ggml_build_forward_order(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) {
-    ggml_build_forward_impl(cgraph, tensor, true, false);
-}
-
 void ggml_build_backward_expand(
         struct ggml_context *  ctx,
         struct ggml_cgraph  *  cgraph,

gguf-py/gguf/scripts/gguf_convert_endian.py

@@ -59,29 +59,11 @@ def byteswap_q6_k(tensor, block_offs):
     delta.byteswap(inplace=True)
 
 
-def byteswap_q1_0(tensor, block_offs):
-    # Each block_q1_0 consists of an f16 delta followed by 16 int8 quantizations.
-
-    # Byte-Swap f16 sized delta field
-    delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-
-def byteswap_tq2_0(tensor, block_offs):
-    # Each block_tq2_0 consists of 64 int8 values followed by 1 f16 value.
-
-    # Byte-Swap f16 sized field
-    delta = tensor.data[block_offs + 64:block_offs + 66].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-
 byteswap_tensors = {
-    gguf.GGMLQuantizationType.Q1_0:  byteswap_q1_0,
     gguf.GGMLQuantizationType.Q4_0:  byteswap_q4_0,
     gguf.GGMLQuantizationType.Q8_0:  byteswap_q8_0,
     gguf.GGMLQuantizationType.Q4_K:  byteswap_q4_k,
     gguf.GGMLQuantizationType.Q6_K:  byteswap_q6_k,
-    gguf.GGMLQuantizationType.TQ2_0: byteswap_tq2_0,
     gguf.GGMLQuantizationType.MXFP4: byteswap_noop,
     gguf.GGMLQuantizationType.NVFP4: byteswap_noop,
 }

scripts/sync-ggml.last

@@ -1 +1 @@
-30bf8685ed4eb0a47f2b06229543327749904150
+90951f99af1fbebef3fbdd58ff5b8715b0bb9c43

scripts/sync_vendor.py

@@ -24,31 +24,10 @@ vendor = {
     "https://raw.githubusercontent.com/sheredom/subprocess.h/8671cee1fc09f11a70ce3782a0ee13177c3aa387/subprocess.h": "vendor/sheredom/subprocess.h",
 }
 
-# TODO @ngxson : this is temporary, to be removed in the future
-patches = [
-    # https://github.com/sheredom/subprocess.h/pull/102
-    "vendor/sheredom/patch-bsd.patch",
-    # https://github.com/sheredom/subprocess.h/pull/101
-    "vendor/sheredom/patch-windows-quote-backslash.patch",
-    # https://github.com/sheredom/subprocess.h/pull/104
-    # note: must be applied after patch-bsd.patch, they touch adjacent lines
-    "vendor/sheredom/patch-glibc-older-than-2.29.patch",
-]
-
 for url, filename in vendor.items():
     print(f"downloading {url} to {filename}") # noqa: NP100
     urllib.request.urlretrieve(url, filename)
 
-for patch in patches:
-    print(f"applying {patch}") # noqa: NP100
-    try:
-        subprocess.check_call([
-            "git", "apply", "--directory", os.path.dirname(patch), patch
-        ])
-    except Exception as e:
-        print(f"Error: {e}") # noqa: NP100
-        sys.exit(1)
-
 print("Splitting httplib.h...") # noqa: NP100
 try:
     subprocess.check_call([

src/llama-arch.cpp

@@ -1014,7 +1014,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_OLMOE:
         case LLM_ARCH_DEEPSEEK2:
         case LLM_ARCH_DEEPSEEK32:
-        case LLM_ARCH_DEEPSEEK4:
         case LLM_ARCH_GLM_DSA:
         case LLM_ARCH_BITNET:
         case LLM_ARCH_T5:

src/llama-grammar.cpp

@@ -648,12 +648,10 @@ const char * llama_grammar_parser::parse_sequence(
             } else {
                 throw std::runtime_error(std::string("expecting ',' at ") + pos);
             }
-            if (min_times > MAX_REPETITION_THRESHOLD) {
+            bool has_max = max_times != UINT64_MAX;
+            if (min_times > MAX_REPETITION_THRESHOLD || (has_max && max_times > MAX_REPETITION_THRESHOLD)) {
                 throw std::runtime_error(std::string("number of repetitions exceeds sane defaults, please reduce the number of repetitions"));
             }
-            if (max_times != UINT64_MAX && max_times > MAX_REPETITION_THRESHOLD) {
-                max_times = UINT64_MAX;
-            }
             handle_repetitions(min_times, max_times);
         } else {
             break;

src/llama-model-loader.cpp

@@ -1249,13 +1249,7 @@ struct ggml_tensor * llama_model_loader::create_tensor(
         for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
             t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
             GGML_ASSERT(t_meta.ne[dim] >= 1);
-            if (dim == 0) {
-                t_meta.nb[dim] = ggml_type_size(type);
-            } else if (dim == 1) {
-                t_meta.nb[dim] = ggml_row_size(type, t_meta.ne[dim-1]);
-            } else {
-                t_meta.nb[dim] = t_meta.nb[dim-1]*t_meta.ne[dim-1];
-            }
+            t_meta.nb[dim] = dim == 0 ? ggml_type_size(type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
             GGML_ASSERT(t_meta.nb[dim] >= 1);
         }
         ggml_set_name(&t_meta, tn.str().c_str());
@@ -1278,18 +1272,10 @@ struct ggml_tensor * llama_model_loader::create_tensor(
     if (flags & TENSOR_ALLOW_RESHAPE) {
         for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
             t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
-            if (dim == 0) {
-                t_meta.nb[dim] = ggml_type_size(t_meta.type);
-            } else if (dim == 1) {
-                t_meta.nb[dim] = ggml_row_size(t_meta.type, t_meta.ne[dim-1]);
-            } else {
-                t_meta.nb[dim] = t_meta.ne[dim-1]*t_meta.nb[dim-1];
-            }
+            t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
         }
     }
 
-    GGML_ASSERT(ggml_nbytes(&t_meta) == ggml_nbytes(cur));
-
     ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta);
     if (buft == nullptr) {
         return nullptr;

src/llama-model.cpp

@@ -355,9 +355,13 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
     static const std::regex pattern_qkv_bias        ("blk\\.\\d*\\.attn_qkv.bias");
     static const std::regex pattern_qk_norm         ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
     static const std::regex pattern_kv_cache        ("cache_(k|v)_l\\d*");
+    static const std::regex pattern_dsv4_state      ("dsv4_(csa|hca|lid)_state_(kv|score)_l\\d*");
     static const std::regex pattern_attn_sinks      ("blk\\.\\d*\\.attn_sinks.weight");
     static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
     static const std::regex pattern_attn_out_bias   ("blk\\.\\d*\\.attn_output.bias");
+    static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight");
+    static const std::regex pattern_attn_out_b_weight("blk\\.\\d*\\.attn_output_b\\.weight");
+    static const std::regex pattern_attn_q_b_weight ("blk\\.\\d*\\.attn_q_b\\.weight");
     static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");
 
     static const std::regex pattern_ssm_dt          ("blk\\.\\d*\\.ssm_dt.bias");
@@ -376,8 +380,11 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
     static const std::regex pattern_ffn_gate_bias     ("blk\\.\\d*\\.ffn_gate(_exps)?.bias");
     static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
     static const std::regex pattern_ffn_down_weight   ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
-    static const std::regex pattern_ffn_down_bias     ("blk\\.\\d*\\.ffn_down.bias");
-    static const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");
+    static const std::regex pattern_ffn_down_bias         ("blk\\.\\d*\\.ffn_down.bias");
+    static const std::regex pattern_ffn_down_exps_bias    ("blk\\.\\d*\\.ffn_down_exps.bias");
+    static const std::regex pattern_ffn_up_shexp_weight   ("blk\\.\\d*\\.ffn_up_shexp.weight");
+    static const std::regex pattern_ffn_gate_shexp_weight ("blk\\.\\d*\\.ffn_gate_shexp.weight");
+    static const std::regex pattern_ffn_down_shexp_weight ("blk\\.\\d*\\.ffn_down_shexp.weight");
 
     static const std::regex pattern_output_weight("output\\.weight");
     static const std::regex pattern_output_bias  ("output\\.bias");
@@ -434,6 +441,37 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
     };
 
     auto get_tensor_config = [&]() -> tensor_config {
+        // dflash drafters are small, mirror them on every device: no reduction boundaries,
+        // and the target hidden-state handoff stays within the same backends
+        if (ud->model->arch == LLM_ARCH_DFLASH) {
+            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+        }
+        if (ud->model->arch == LLM_ARCH_DEEPSEEK4) {
+            if (std::regex_match(tensor_name, pattern_kv_cache) ||
+                    std::regex_match(tensor_name, pattern_dsv4_state)) {
+                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+            }
+            if (std::regex_match(tensor_name, pattern_attn_sinks)) {
+                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output_a.weight");
+            }
+            if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) {
+                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output_a.weight");
+            }
+            if (std::regex_match(tensor_name, pattern_attn_out_a_weight)) {
+                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_2, "attn_output_b.weight");
+            }
+            if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) {
+                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
+            }
+            if (std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) ||
+                    std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight)) {
+                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down_shexp.weight");
+            }
+            if (std::regex_match(tensor_name, pattern_ffn_down_shexp_weight)) {
+                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down_shexp.weight");
+            }
+        }
+
         // standard attention
         if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight)) {
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");
@@ -621,9 +659,29 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
 
             if (std::regex_match(tensor_name, pattern_attn_sinks)) {
                 GGML_ASSERT(segments.size() == 1);
+                if (ud->model->arch == LLM_ARCH_DEEPSEEK4) {
+                    return {hparams.n_head(il) / hparams.dsv4_o_group_count};
+                }
                 return {std::lcm(n_embd_q, blck_size_perf)/n_embd_q * n_gqa};
             }
 
+            if (ud->model->arch == LLM_ARCH_DEEPSEEK4) {
+                if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) {
+                    GGML_ASSERT(segments.size() == 1);
+                    // the grouped output projection requires each device to hold whole groups of heads
+                    const int64_t n_head_group = hparams.n_head(il) / hparams.dsv4_o_group_count;
+                    return {n_head_group * hparams.n_embd_head_k(il)};
+                }
+                if (std::regex_match(tensor_name, pattern_attn_out_a_weight)) {
+                    GGML_ASSERT(segments.size() == 1);
+                    return {1};
+                }
+                if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) {
+                    GGML_ASSERT(segments.size() == 1);
+                    return {std::lcm<int64_t>(hparams.dsv4_o_lora_rank, blck_size)};
+                }
+            }
+
             const int64_t granularity_q = std::lcm(n_embd_q, blck_size_perf);
             if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) {
                 GGML_ASSERT(segments.size() == 1);
@@ -654,7 +712,11 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
         // FFN
         if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) ||
                 std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) ||
-                std::regex_match(tensor_name, pattern_ffn_gate_up_weight) || std::regex_match(tensor_name, pattern_ffn_down_weight)) {
+                std::regex_match(tensor_name, pattern_ffn_gate_up_weight) ||
+                std::regex_match(tensor_name, pattern_ffn_down_weight) ||
+                std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) ||
+                std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight) ||
+                std::regex_match(tensor_name, pattern_ffn_down_shexp_weight)) {
             const int64_t blck_size_perf = std::lcm(blck_size, 128);
             GGML_ASSERT(segments.size() == 1);
             return {blck_size_perf};
@@ -2890,21 +2952,6 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,
         int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
         int flags) {
     const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_;
-
-    if (flags & TENSOR_SKIP) {
-        const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP;
-
-        create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
-        create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias",   bid), {n_embd_qkv},          skip | TENSOR_SKIP_IF_VIRTUAL);
-        create_tensor(tn(LLM_TENSOR_ATTN_Q,   "weight", bid), {n_embd_, n_embd_q_},  skip);
-        create_tensor(tn(LLM_TENSOR_ATTN_K,   "weight", bid), {n_embd_, n_embd_k_},  skip);
-        create_tensor(tn(LLM_TENSOR_ATTN_V,   "weight", bid), {n_embd_, n_embd_v_},  skip);
-        create_tensor(tn(LLM_TENSOR_ATTN_Q,   "bias",   bid), {n_embd_q_},           skip);
-        create_tensor(tn(LLM_TENSOR_ATTN_K,   "bias",   bid), {n_embd_k_},           skip);
-        create_tensor(tn(LLM_TENSOR_ATTN_V,   "bias",   bid), {n_embd_v_},           skip);
-        return;
-    }
-
     layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
     if (layer.wqkv) {
         layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);

tests/test-backend-ops.cpp

@@ -9747,15 +9747,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
 static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
     std::vector<std::unique_ptr<test_case>> test_cases;
 
-    // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands
-    // note: same bytes either way, so a backend that indexes them differently shows it here
-    for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
-        for (int64_t n_tokens : {512, 2048}) {
-            test_cases.emplace_back(new test_glu(GGML_GLU_OP_SWIGLU, type, { 2*17408, n_tokens, 1, 1 }, 0, false));
-            test_cases.emplace_back(new test_glu_split(GGML_GLU_OP_SWIGLU, type, { 17408, n_tokens, 1, 1 }, 0));
-        }
-    }
-
     // Conv2d: K=CRS=NPQ=4096 matmul performance
     uint32_t                        iwh_idx  = 0;
     uint32_t                        kwh_idx  = 1;

tests/test-backend-sampler.cpp

@@ -1668,18 +1668,9 @@ static std::vector<const backend_test_case *> collect_tests_to_run(const std::st
         }
     } else {
         for (const auto & test : BACKEND_TESTS) {
-            if (!test.enabled_by_default) {
-                continue;
+            if (test.enabled_by_default) {
+                selected.push_back(&test);
             }
-#ifdef GGML_USE_HIP
-            // TODO: remove this when https://github.com/ggml-org/llama.cpp/pull/26592 is merged
-            if (test.name == "penalties" || test.name == "set_sampler" ||
-                test.name == "mixed"     || test.name == "top_p") {
-                fprintf(stderr, "Skipping test '%s' on HIP backend (no backend TOP_K support)\n", test.name.c_str());
-                continue;
-            }
-#endif // GGML_USE_HIP
-            selected.push_back(&test);
         }
     }
 

tests/test-grammar-parser.cpp

@@ -153,53 +153,6 @@ int main()
         root ::= "a"{,10}"
     )""");
 
-    verify_failure(R"""(
-        root ::= "a"{5000}
-    )""");
-
-    verify_failure(R"""(
-        root ::= "a"{5000,}
-    )""");
-
-    verify_failure(R"""(
-        root ::= "a"{5000,6000}
-    )""");
-
-    verify_parsing(R"""(
-        root ::= "a"{0,5000}
-    )""", {
-        {"root", 0},
-        {"root_1", 1},
-    }, {
-        // root (index 0)
-        {LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
-        {LLAMA_GRETYPE_END, 0},
-        // root_1 (index 1)
-        {LLAMA_GRETYPE_CHAR, 'a'},
-        {LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
-        {LLAMA_GRETYPE_ALT, 0},
-        {LLAMA_GRETYPE_END, 0},
-    });
-
-    verify_parsing(R"""(
-        root ::= "a"{3,5000}
-    )""", {
-        {"root", 0},
-        {"root_1", 1},
-    }, {
-        // root (index 0)
-        {LLAMA_GRETYPE_CHAR, 'a'},
-        {LLAMA_GRETYPE_CHAR, 'a'},
-        {LLAMA_GRETYPE_CHAR, 'a'},
-        {LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
-        {LLAMA_GRETYPE_END, 0},
-        // root_1 (index 1)
-        {LLAMA_GRETYPE_CHAR, 'a'},
-        {LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
-        {LLAMA_GRETYPE_ALT, 0},
-        {LLAMA_GRETYPE_END, 0},
-    });
-
     verify_parsing(R"""(
         root  ::= "a"
     )""", {

tests/test-llama-archs.cpp

@@ -101,6 +101,12 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         n_head = 1;
         n_ff   = 96;
         n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
+    } else if (arch == LLM_ARCH_DEEPSEEK4) {
+        // head size 64 so that GPU flash attention kernels support the model
+        n_embd  = 512;
+        n_head  = 8;
+        n_ff    = 1024;
+        n_layer = 4;
     } else if (arch == LLM_ARCH_DEEPSEEK2
             || arch == LLM_ARCH_DEEPSEEK32
             || arch == LLM_ARCH_GLM_DSA
@@ -156,11 +162,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);
     } else {
         ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
-        ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head);
+        ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(1) : n_head);
     }
 
     ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);
-    if (arch == LLM_ARCH_DEEPSEEK2
+    if (arch == LLM_ARCH_DEEPSEEK4) {
+        ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH,   n_embd_head);
+        ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head);
+        ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT,   n_embd_head/2);
+    } else if (arch == LLM_ARCH_DEEPSEEK2
             || arch == LLM_ARCH_DEEPSEEK32
             || arch == LLM_ARCH_GLM_DSA
             || arch == LLM_ARCH_KIMI_LINEAR
@@ -179,7 +189,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,      1e-5f);
     ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS,          1e-5f);
     ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS,       uint32_t(8));
-    ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK,            uint32_t(512));
+    ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK,            arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(64) : uint32_t(512));
     ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK,           uint32_t(512));
     ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8));
     ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW,         n_ctx/8);
@@ -205,12 +215,26 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
 
     // MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the
     // indexer head count is independent of the main attention head count.
-    ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,   arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(1));
+    ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,   arch == LLM_ARCH_MINIMAX_M3 || arch == LLM_ARCH_DEEPSEEK4 ? n_head : uint32_t(1));
     ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,   uint32_t(64));
     ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K,        uint32_t(8));
     ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE,   uint32_t(4));
     ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));
     ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));
+
+    if (arch == LLM_ARCH_DEEPSEEK4) {
+        ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,         uint32_t(8));
+        ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK,           uint32_t(32));
+        ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS,            std::vector<uint32_t>({0, 0, 4, 128}));
+        ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE,    160000.0f);
+        ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT,               uint32_t(4));
+        ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
+        ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON,             1.0e-6f);
+        ms.add_kv(LLM_KV_HASH_LAYER_COUNT,                      uint32_t(0));
+        ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP,                      10.0f);
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE,                  1.0f);
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM,                   true);
+    }
     ms.add_kv(LLM_KV_TOKENIZER_MODEL,         "no_vocab");
     // ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT,     n_embd);
     // ms.add_kv(LLM_KV_DENSE_3_FEAT_IN,      n_embd);
@@ -221,7 +245,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         ms.add_kv(LLM_KV_EXPERT_COUNT,               uint32_t(2));
         ms.add_kv(LLM_KV_EXPERT_USED_COUNT,          uint32_t(1));
         ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT,        uint32_t(1));
-        ms.add_kv(LLM_KV_EXPERT_GATING_FUNC,         uint32_t(2)); // sigmoid
+        ms.add_kv(LLM_KV_EXPERT_GATING_FUNC,         arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2));
         ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE,         1.0f);
         ms.add_kv(LLM_KV_EXPERTS_PER_GROUP,          uint32_t(1));
     }
@@ -347,6 +371,7 @@ static bool moe_mandatory(const llm_arch arch) {
         case LLM_ARCH_DEEPSEEK:
         case LLM_ARCH_DEEPSEEK2:
         case LLM_ARCH_DEEPSEEK32:
+        case LLM_ARCH_DEEPSEEK4:
         case LLM_ARCH_GLM4_MOE:
         case LLM_ARCH_GLM_DSA:
         case LLM_ARCH_EXAONE_MOE:
@@ -426,25 +451,13 @@ static bool arch_supported(const llm_arch arch) {
     if (arch == LLM_ARCH_DEEPSEEK2OCR) {
         return false;
     }
-    if (arch == LLM_ARCH_DEEPSEEK4) {
-        return false;
-    }
-
     // FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI.
 #ifdef GGML_USE_WEBGPU
-    if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA) {
+    if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MINIMAX_M3) {
         return false;
     }
 #endif // GGML_USE_WEBGPU
 
-    // FIXME: jamba produces incorrect output (~0.55 NMSE vs CPU) on the HIP
-    // backend on RDNA3.5 (gfx1151); the SSM kernels need investigation.
-#ifdef GGML_USE_HIP
-    if (arch == LLM_ARCH_JAMBA) {
-        return false;
-    }
-#endif // GGML_USE_HIP
-
     return true;
 }
 
@@ -614,10 +627,18 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg
                     if (logits_cpu.empty()) {
                         model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {}, LLAMA_SPLIT_MODE_LAYER, encode);
                         logits_cpu = get_logits(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, encode);
+                        if (arch == LLM_ARCH_DEEPSEEK4) {
+                            GGML_ASSERT(llama_memory_seq_rm(
+                                    llama_get_memory(model_and_ctx_cpu.second.get()), 0, -1, -1));
+                        }
                     }
                     if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llm_arch_supports_sm_tensor(arch)) {
                         model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, dc.devs, dc.split_mode, encode);
                         logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);
+                        if (arch == LLM_ARCH_DEEPSEEK4) {
+                            GGML_ASSERT(llama_memory_seq_rm(
+                                    llama_get_memory(model_and_ctx_dev.second.get()), 0, -1, -1));
+                        }
                         const double nmse_val = nmse(logits_cpu, logits_dev);
                         snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);
                         status_nmse = "\033[1;32mOK\033[0m";
@@ -630,7 +651,9 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg
                     FILE * file = tmpfile(); // Can be null on Windows without administrator privileges.
                     // FIXME: when adding a tensor to a gguf_context a copy is made, this changes the pointer which the meta backend
                     //     in turn uses to map the tensors to their simple equivalents - this is fundamentally incompatible
-                    if (file != nullptr && llama_model_saver_supports_arch(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {
+                    // FIXME: DSV4 metadata is not implemented by llama_model_saver.
+                    const bool can_roundtrip = llama_model_saver_supports_arch(arch) && arch != LLM_ARCH_DEEPSEEK4;
+                    if (file != nullptr && can_roundtrip && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {
                         GGML_ASSERT(model_and_ctx_dev.first && model_and_ctx_dev.second);
                         llama_model_saver ms = llama_model_saver(model_and_ctx_dev.first.get());
                         ms.add_kv_from_model();

tests/test-model-resolution.cpp

@@ -195,7 +195,7 @@ static const std::vector<std::string> dspark_dflash = {
 
 struct plan_case {
     const char * name;
-    const std::vector<std::string> files;
+    const std::vector<std::string> & files;
     const char * hf_repo;
     const char * hf_file;
     bool sidecars;        // request mmproj + mtp + dflash + eagle3 + dspark

tests/test-mtmd-c-api.c

@@ -1,6 +1,4 @@
 #include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
 #include <assert.h>
 
 #include "mtmd.h"
@@ -64,72 +62,6 @@ int main(void) {
         }
     }
 
-    // test chunk save/load round-trip
-    for (size_t i = 0; i < n_chunks; i++) {
-        const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i);
-        assert(chunk != NULL);
-        enum mtmd_input_chunk_type type = mtmd_input_chunk_get_type(chunk);
-
-        // query the required buffer size (out_buf == NULL)
-        size_t expected_len = 0;
-        int32_t rc = mtmd_input_chunk_save(chunk, NULL, 0, &expected_len);
-        printf("    Chunk %zu: save query rc = %d, expected_len = %zu\n", i, rc, expected_len);
-        assert(rc == 0);
-        assert(expected_len > 0);
-
-        // saving into a too-small buffer must fail, not crash
-        char tiny_buf[1];
-        rc = mtmd_input_chunk_save(chunk, tiny_buf, sizeof(tiny_buf), NULL);
-        printf("    Chunk %zu: save into too-small buffer rc = %d (expect non-zero)\n", i, rc);
-        assert(rc != 0);
-
-        // save into a properly-sized buffer
-        char * buf = (char *) malloc(expected_len);
-        assert(buf != NULL);
-        rc = mtmd_input_chunk_save(chunk, buf, expected_len, NULL);
-        assert(rc == 0);
-
-        // loading from a truncated buffer must fail gracefully, not crash
-        if (expected_len > 1) {
-            mtmd_input_chunk * bad = mtmd_input_chunk_load(buf, expected_len - 1);
-            printf("    Chunk %zu: load from truncated buffer = %p (expect NULL)\n", i, (void *) bad);
-            assert(bad == NULL);
-        }
-
-        // load it back
-        mtmd_input_chunk * loaded = mtmd_input_chunk_load(buf, expected_len);
-        assert(loaded != NULL);
-
-        // metadata must match the original chunk
-        assert(mtmd_input_chunk_get_type(loaded) == type);
-        assert(mtmd_input_chunk_get_n_tokens(loaded) == mtmd_input_chunk_get_n_tokens(chunk));
-        assert(mtmd_input_chunk_get_n_pos(loaded) == mtmd_input_chunk_get_n_pos(chunk));
-
-        if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
-            size_t n_tok_orig, n_tok_loaded;
-            const llama_token * tok_orig   = mtmd_input_chunk_get_tokens_text(chunk, &n_tok_orig);
-            const llama_token * tok_loaded = mtmd_input_chunk_get_tokens_text(loaded, &n_tok_loaded);
-            printf("    Chunk %zu: loaded %zu text tokens (orig %zu), first token %d (orig %d)\n",
-                i, n_tok_loaded, n_tok_orig,
-                n_tok_loaded > 0 ? tok_loaded[0] : -1,
-                n_tok_orig   > 0 ? tok_orig[0]   : -1);
-            assert(n_tok_orig == n_tok_loaded);
-            for (size_t j = 0; j < n_tok_orig; j++) {
-                assert(tok_orig[j] == tok_loaded[j]);
-            }
-        } else if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
-            const char * id_orig   = mtmd_input_chunk_get_id(chunk);
-            const char * id_loaded = mtmd_input_chunk_get_id(loaded);
-            printf("    Chunk %zu: loaded id '%s' (orig '%s')\n", i, id_loaded, id_orig);
-            assert(id_orig != NULL && id_loaded != NULL);
-            assert(strcmp(id_orig, id_loaded) == 0);
-        }
-
-        mtmd_input_chunk_free(loaded);
-        free(buf);
-    }
-    printf("Chunk save/load round-trip OK\n");
-
     // Free the chunks
     mtmd_input_chunks_free(chunks);
 

tools/mtmd/clip-impl.h

@@ -591,8 +591,6 @@ struct clip_image_u8 {
     }
 };
 
-struct mtmd_serialization; // forward declaration
-
 // For images, buf.size() == nx*ny*3
 //     Memory layout: RGBRGBRGB...
 // For seq, buf.size() == nx*ny*3*nt
@@ -673,9 +671,6 @@ struct clip_image_f32 {
         return buf.empty();
     }
 
-    void serialize(struct mtmd_serialization & ser) const;
-    void deserialize(struct mtmd_serialization & ser);
-
   private:
     std::vector<float> buf;
     int nx_ = 0;
@@ -757,9 +752,6 @@ struct clip_image_f32_batch {
         }
         return new_batch;
     }
-
-    void serialize(struct mtmd_serialization & ser) const;
-    void deserialize(struct mtmd_serialization & ser);
 };
 
 //

tools/mtmd/clip-model.h

@@ -170,17 +170,6 @@ struct clip_hparams {
         warmup_image_size = static_cast<int>(std::sqrt(image_max_pixels));
     }
 
-    // used by longest_edge preprocessor (no model-specific value for min/max tokens)
-    void set_limit_image_tokens() {
-        const int patch_area = patch_size * patch_size * n_merge * n_merge;
-        if (custom_image_min_tokens > 0) {
-            image_min_pixels = custom_image_min_tokens * patch_area;
-        }
-        if (custom_image_max_tokens > 0) {
-            image_max_pixels = custom_image_max_tokens * patch_area;
-        }
-    }
-
     void set_warmup_n_tokens(int n_tokens) {
         int n_tok_per_side = static_cast<int>(std::sqrt(n_tokens));
         GGML_ASSERT(n_tok_per_side * n_tok_per_side == n_tokens && "n_tokens must be n*n");

tools/mtmd/clip.cpp

@@ -708,10 +708,9 @@ ggml_tensor * clip_graph::build_attn(
         ggml_tensor * sinks) const {
     // these nodes are added to the graph together so that they are not reordered
     // by doing so, the number of splits in the graph is reduced
-    // the order is fixed without the compute flag, so an unselected branch stays out of the compute set
-    ggml_build_forward_order(gf, q_cur);
-    ggml_build_forward_order(gf, k_cur);
-    ggml_build_forward_order(gf, v_cur);
+    ggml_build_forward_expand(gf, q_cur);
+    ggml_build_forward_expand(gf, k_cur);
+    ggml_build_forward_expand(gf, v_cur);
 
     ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3);
     //cb(q, "q", il);
@@ -1434,7 +1433,6 @@ struct clip_model_loader {
                         // use default llava-uhd preprocessing params
                         get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
                         get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
-                        hparams.set_limit_image_tokens();
                     } break;
                 case PROJECTOR_TYPE_LFM2:
                     {
@@ -1472,7 +1470,6 @@ struct clip_model_loader {
                         get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
                         hparams.image_longest_edge = hparams.image_size;
                         get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
-                        hparams.set_limit_image_tokens();
                         hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
                     } break;
                 case PROJECTOR_TYPE_DOTS_OCR:
@@ -1597,7 +1594,6 @@ struct clip_model_loader {
                         if (hparams.image_longest_edge == 0) {
                             hparams.image_longest_edge = 3024;
                         }
-                        // note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokens
                         hparams.warmup_image_size = hparams.image_size;
                     } break;
                 case PROJECTOR_TYPE_YOUTUVL:
@@ -1765,10 +1761,6 @@ struct clip_model_loader {
                             // qwen2 encoder is GQA, requires KEY_N_HEAD_KV
                             get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
                         }
-                        // unlimited-ocr shares the v1 projector but tiles up to 32
-                        get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
-                        get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
-                        GGML_ASSERT(hparams.preproc_min_tiles <= hparams.preproc_max_tiles);
                      } break;
                 case PROJECTOR_TYPE_HUNYUANVL:
                     {
@@ -1917,9 +1909,6 @@ struct clip_model_loader {
                 if (hparams.image_max_pixels > 0) {
                     LOG_INF("%s: image_max_pixels:   %d%s\n", __func__, hparams.image_max_pixels, hparams.custom_image_max_tokens > 0 ? " (custom value)" : "");
                 }
-                if (hparams.preproc_max_tiles > 0) {
-                    LOG_INF("%s: preproc_tiles:      %d - %d\n", __func__, hparams.preproc_min_tiles, hparams.preproc_max_tiles);
-                }
             } else if (is_audio) {
                 LOG_INF("\n--- audio hparams ---\n");
                 LOG_INF("%s: n_mel_bins:         %d\n", __func__, hparams.n_mel_bins);

tools/mtmd/models/deepseekocr.cpp

@@ -253,9 +253,6 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
 
     bool is_overview = img.add_viewsep;
     int n_tiles_per_row = 0;
-    // number of separate "row" images batched together in this graph call
-    // (captured now, before n_batch below gets repurposed as the SAM/ViT batch size)
-    const int n_rows_batch = n_batch;
 
     // note: we expect either a batch of rows or a batch of overviews, but not a mix of both
 
@@ -275,18 +272,16 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
         GGML_ASSERT(img.ny() % img.nx() == 0);
         n_tiles_per_row = img.ny() / img.nx();
 
-        // each entry is one "row" image of shape [tile_size, tile_size * n_tiles_per_row, 3];
-        // merge the tile axis into the batch axis, giving a combined SAM input of shape
-        // [tile_size, tile_size, 3, n_tiles_per_row * n_rows_batch] (tile fast, row slow)
-        inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx() * img.nx(), n_tiles_per_row, 3, n_rows_batch);
-        inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 2, 1, 3));
-        inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), 3, n_tiles_per_row * n_rows_batch);
+        // input shape: [tile_size, tile_size * n_tiles_per_row, 3]
+        // we want to reshape it to [tile_size, tile_size, 3, n_tiles_per_row]
+        inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), n_tiles_per_row, 3);
+        inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 1, 3, 2));
     }
 
     ggml_tensor * sam_out = build_sam(inp_raw);
 
     if (!is_overview) {
-        n_batch = n_tiles_per_row * n_rows_batch;
+        n_batch = n_tiles_per_row;
     }
 
     const int clip_n_patches = sam_out->ne[0] * sam_out->ne[1];
@@ -359,36 +354,34 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
         const auto w     = h;
         const auto n_dim = cur->ne[0];
 
-        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, n_batch);
-        cur = ggml_reshape_4d(ctx0, cur, n_dim, w, h, n_batch);
-        cur = ggml_reshape_3d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h, n_batch);
-        ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, n_dim, 1, n_batch, 1);
-        cur = ggml_concat(ctx0, cur, vs, 1);  // (n_dim, h*(w+1) + 1, n_batch)
+        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
+        cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
+        cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
+        cur = ggml_concat(ctx0, cur, model.view_seperator, 1);  // (n_dim, h*(w+1) + 1)
     } else {
         // tile row: interleave tiles within each row, add newline per row
-        const int  grid_x = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
-        const int  grid_y = grid_x;
-        const auto n_dim  = cur->ne[0];
+        const int grid_x      = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
+        const int grid_y      = grid_x;
+        const auto n_dim      = cur->ne[0];
 
-        // merge n_dim into the grid_x axis, freeing the 4th axis for n_rows_batch
-        // (n_dim, clip_n_patches, n_tiles_per_row * n_rows_batch) -> (n_dim*grid_x, grid_y, n_tiles_per_row, n_rows_batch)
-        cur = ggml_reshape_4d(ctx0, cur, n_dim * grid_x, grid_y, n_tiles_per_row, n_rows_batch);
+        // (n_dim, clip_n_patches, n_batch) -> (n_dim, grid_x, grid_y, n_batch)
+        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x, grid_y, n_batch);
 
         // tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
         //        to A.row0 B.row0 A.row1 B.row1 ...
         //        then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
-        // interleave tiles: -> (n_dim*grid_x, n_tiles_per_row, grid_y, n_rows_batch)
-        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
+        // interleave tiles: (n_dim, grid_x, grid_y, n_batch) -> (n_dim, grid_x, n_batch, grid_y)
+        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2));
 
-        // merge: -> (n_dim, grid_x*n_tiles_per_row, grid_y, n_rows_batch)
-        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_tiles_per_row, grid_y, n_rows_batch);
+        // merge: (n_dim, grid_x, n_batch, grid_y) -> (n_dim, grid_x*n_batch, grid_y, 1)
+        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_batch, grid_y, 1);
 
-        // append newline per row: (n_dim, grid_x*n_tiles_per_row+1, grid_y, n_rows_batch)
-        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, n_rows_batch);
+        // append newline per row: (n_dim, grid_x*n_batch+1, grid_y, 1)
+        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, 1);
         cur = ggml_concat(ctx0, cur, imgnl, 1);
 
-        // flatten: (n_dim, (grid_x*n_tiles_per_row+1)*grid_y, n_rows_batch)
-        cur = ggml_reshape_3d(ctx0, cur, n_dim, (grid_x * n_tiles_per_row + 1) * grid_y, n_rows_batch);
+        // flatten: (n_dim, (grid_x*n_batch+1)*grid_y)
+        cur = ggml_reshape_2d(ctx0, cur, n_dim, (grid_x * n_batch + 1) * grid_y);
     }
 
     cb(cur, "dsocr_output", -1);

tools/mtmd/models/deepseekocr2.cpp

@@ -14,9 +14,8 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
     {
         ggml_tensor * inp;
 
-        // H*W, C, B
-        inp = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2], sam_out->ne[3]);
-        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); // C, H*W, B
+        inp = ggml_reshape_2d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2]); // H*W, C
+        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
 
         auto num_image_tokens = inp->ne[1]; // H*W
         GGML_ASSERT(num_image_tokens == 144 || num_image_tokens == 256);
@@ -33,10 +32,8 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
             num_queries = 144;
         }
 
-        // repeat the query embedding per batch item, then append: (C, num_image_tokens + num_queries, B)
-        query_embed = ggml_cast(ctx0, query_embed, inp->type);
-        query_embed = ggml_repeat_4d(ctx0, query_embed, query_embed->ne[0], num_queries, inp->ne[2], 1);
-        inp = ggml_concat(ctx0, inp, query_embed, 1);
+        // (B, num_image_tokens + num_queries, C)
+        inp = ggml_concat(ctx0, inp, ggml_cast(ctx0, query_embed, inp->type), 1);
 
         auto seq_len = inp->ne[1];
 
@@ -60,7 +57,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
                                       /* learned_pos_embd */ nullptr, add_rope, vit_opts);
 
         cur = ggml_cont(ctx0,
-                        ggml_view_3d(ctx0, cur, cur->ne[0], num_queries, cur->ne[2], cur->nb[1], cur->nb[2],
+                        ggml_view_2d(ctx0, cur, cur->ne[0], num_queries, cur->nb[1],
                                      cur->nb[1] * (cur->ne[1] - num_queries))); // only take query tokens for output
 
         ggml_build_forward_expand(gf, cur);
@@ -74,8 +71,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
 
     // view_seperator only after the global view
     if (img.add_viewsep) {
-        ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, model.view_seperator->ne[0], 1, cur->ne[2], 1);
-        cur = ggml_concat(ctx0, cur, vs, 1); // (n_dim, 257, n_batch)
+        cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, 257)
     }
 
     cb(cur, "dsocr2_output", -1);

tools/mtmd/models/models.h

@@ -138,13 +138,12 @@ struct clip_graph_deepseekocr : clip_graph {
     clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
     ggml_cgraph * build() override;
     ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
-    bool support_batch() const override { return true; }
+    // bool support_batch() const override { return true; } // TODO: support batch for DeepSeek-OCR v1
 };
 
 struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
     clip_graph_deepseekocr2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_deepseekocr(ctx, img) {}
     ggml_cgraph * build() override; // reuses build_sam() from base
-    bool support_batch() const override { return true; }
 };
 
 struct clip_graph_conformer : clip_graph {

tools/mtmd/mtmd-helper-gen.cpp

@@ -112,6 +112,7 @@ public:
         c2w_state.clear();
         audio_pcm.clear();
         overlay.clear();
+        overlay_idx = 0;
         h_state_buf.clear();
         out_buf.clear();
         prompt_embd_buf.clear();
@@ -204,9 +205,11 @@ public:
         top_p = inp->top_p > 0 ? inp->top_p : 1.0f;
         out_type = inp->out_type;
 
-        // the prompt above holds the whole text stream up to tts_eos, so every generated
-        // frame adds tts_pad on top of the codes embedding
-        overlay = row(tts_pad);
+        // the text stream keeps flowing during generation: after frame k, the input adds
+        // trailing text row k on top of the codes embedding, then tts_eos, then tts_pad
+        for (int i = 3; i < n_ids - 5; i++) overlay.push_back(row(ids[(size_t) i]));
+        overlay.push_back(row(tts_eos));
+        overlay.push_back(row(tts_pad));
 
         return 0;
     }
@@ -262,7 +265,9 @@ public:
         }
 
         std::vector<float> fb(out.embd, out.embd + n_embd);
-        for (int i = 0; i < n_embd; i++) fb[(size_t) i] += overlay[(size_t) i];
+        const auto & ov = overlay[std::min(overlay_idx, overlay.size() - 1)];
+        for (int i = 0; i < n_embd; i++) fb[(size_t) i] += ov[(size_t) i];
+        overlay_idx++;
 
         const int n_pos_per_embd = mrope ? 4 : 1;
         decode_embd_batch batch_embd(fb.data(), 1, n_pos_per_embd, n_embd);
@@ -432,7 +437,8 @@ private:
     std::vector<int32_t> codes_buf;
     std::vector<uint8_t> c2w_state;
     std::vector<float>   audio_pcm;
-    std::vector<float> overlay;
+    std::vector<std::vector<float>> overlay;
+    size_t overlay_idx = 0;
     std::vector<float> h_state_buf;
     mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
     std::vector<char> out_buf;

tools/mtmd/mtmd-image.cpp

@@ -139,46 +139,50 @@ struct img_tool {
         }
     }
 
-    struct calc_size_opt {
-        int align_size = 1;
-        int min_pixels = 0;   // 0 = disabled
-        int max_pixels = 0;   // 0 = disabled
-        // applied before min/max_pixels, so min_pixels can push an edge back above longest_edge
-        int longest_edge = 0; // 0 = disabled
-    };
-
-    // calculate the size of the **resized** image, while preserving the aspect ratio and
-    // aligning to the nearest multiple of align_size ("smart_resize" in transformers code)
-    static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const calc_size_opt & opts) {
-        GGML_ASSERT(opts.align_size > 0);
-        const int width  = inp_size.width;
-        const int height = inp_size.height;
-        if (width <= 0 || height <= 0) {
+    // calculate the size of the **resized** image, while preserving the aspect ratio
+    // the calculated size will be aligned to the nearest multiple of align_size
+    // if H or W size is larger than longest_edge, it will be resized to longest_edge
+    static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int longest_edge) {
+        GGML_ASSERT(align_size > 0);
+        if (inp_size.width <= 0 || inp_size.height <= 0 || longest_edge <= 0) {
             return {0, 0};
         }
 
-        auto round_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };
-        auto ceil_by_factor  = [f = opts.align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };
-        auto floor_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };
+        float scale = std::min(static_cast<float>(longest_edge) / inp_size.width,
+                               static_cast<float>(longest_edge) / inp_size.height);
 
-        int w_bar, h_bar;
-        if (opts.longest_edge > 0) {
-            const float scale = std::min(static_cast<float>(opts.longest_edge) / width,
-                                          static_cast<float>(opts.longest_edge) / height);
-            w_bar = ceil_by_factor(width  * scale);
-            h_bar = ceil_by_factor(height * scale);
-        } else {
-            // always align up first
-            w_bar = std::max(opts.align_size, round_by_factor(width));
-            h_bar = std::max(opts.align_size, round_by_factor(height));
-        }
+        float target_width_f  = static_cast<float>(inp_size.width)  * scale;
+        float target_height_f = static_cast<float>(inp_size.height) * scale;
 
-        if (opts.max_pixels > 0 && h_bar * w_bar > opts.max_pixels) {
-            const auto beta = std::sqrt(static_cast<float>(height) * width / opts.max_pixels);
-            h_bar = std::max(opts.align_size, floor_by_factor(height / beta));
-            w_bar = std::max(opts.align_size, floor_by_factor(width  / beta));
-        } else if (opts.min_pixels > 0 && h_bar * w_bar < opts.min_pixels) {
-            const auto beta = std::sqrt(static_cast<float>(opts.min_pixels) / (static_cast<float>(height) * width));
+        auto ceil_by_factor = [f = align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };
+        int aligned_width  = ceil_by_factor(target_width_f);
+        int aligned_height = ceil_by_factor(target_height_f);
+
+        return {aligned_width, aligned_height};
+    }
+
+    // calculate the size of the **resized** image, while preserving the aspect ratio
+    // the calculated size will have min_pixels <= W*H <= max_pixels
+    // this is referred as "smart_resize" in transformers code
+    static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int min_pixels, const int max_pixels) {
+        GGML_ASSERT(align_size > 0);
+        const int width  = inp_size.width;
+        const int height = inp_size.height;
+
+        auto round_by_factor = [f = align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };
+        auto ceil_by_factor  = [f = align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };
+        auto floor_by_factor = [f = align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };
+
+        // always align up first
+        int h_bar = std::max(align_size, round_by_factor(height));
+        int w_bar = std::max(align_size, round_by_factor(width));
+
+        if (h_bar * w_bar > max_pixels) {
+            const auto beta = std::sqrt(static_cast<float>(height * width) / max_pixels);
+            h_bar = std::max(align_size, floor_by_factor(height / beta));
+            w_bar = std::max(align_size, floor_by_factor(width  / beta));
+        } else if (h_bar * w_bar < min_pixels) {
+            const auto beta = std::sqrt(static_cast<float>(min_pixels) / (height * width));
             h_bar = ceil_by_factor(height * beta);
             w_bar = ceil_by_factor(width * beta);
         }
@@ -933,12 +937,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
     const int cur_merge = hparams.n_merge;
     const clip_image_size target_size = img_tool::calc_size_preserved_ratio(
         original_size,
-        {
-            /* align_size   */ hparams.patch_size * cur_merge,
-            /* min_pixels   */ hparams.image_min_pixels,
-            /* max_pixels   */ hparams.image_max_pixels,
-            /* longest_edge */ 0,
-        });
+        hparams.patch_size * cur_merge,
+        hparams.image_min_pixels,
+        hparams.image_max_pixels);
     img_tool::resize(img, resized_image, target_size,
                         hparams.image_resize_algo,
                         hparams.image_resize_pad,
@@ -960,12 +961,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl
     const int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge;
     const clip_image_size target_size = img_tool::calc_size_preserved_ratio(
         original_size,
-        {
-            /* align_size   */ hparams.patch_size * cur_merge,
-            /* min_pixels   */ std::max(0, hparams.image_min_pixels),
-            /* max_pixels   */ std::max(0, hparams.image_max_pixels),
-            /* longest_edge */ hparams.image_longest_edge,
-        });
+        hparams.patch_size * cur_merge,
+        hparams.image_longest_edge);
     img_tool::resize(img, resized_image, target_size,
                         hparams.image_resize_algo,
                         hparams.image_resize_pad,
@@ -1003,8 +1000,8 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
     mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
     const int align_size = hparams.patch_size * hparams.n_merge;
     inst.overview_size = img_tool::calc_size_preserved_ratio(
-                            original_size,
-                            { align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });
+                            original_size, align_size,
+                            hparams.image_min_pixels, hparams.image_max_pixels);
     // tile if either dimension exceeds tile_size with tolerance
     const bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance;
 
@@ -1112,8 +1109,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
     // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737
     const clip_image_size original_size = img.get_size();
     const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(
-        original_size,
-        { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });
+        original_size, hparams.image_size, hparams.image_longest_edge);
     // LOG_INF("%s: original size: %d x %d, refined size: %d x %d\n",
     //         __func__, original_size.width, original_size.height,
     //         refined_size.width, refined_size.height);

tools/mtmd/mtmd.cpp

@@ -22,123 +22,8 @@
 #include <cstdlib>
 #include <cstring>
 #include <climits>
-#include <type_traits>
 #include <vector>
 
-// remember to bump this if the serialization format changes
-#define MTMD_SERIALIZATION_VERSION 1
-
-struct mtmd_serialization {
-    // note: using 64-bit here for future-proofing
-    uint64_t version = MTMD_SERIALIZATION_VERSION;
-    std::vector<char> data;
-    size_t read_pos = 0; // cursor used when reading
-
-    // for writing
-    mtmd_serialization(uint64_t version) : version(version) {
-        write(version);
-    }
-
-    // for reading
-    mtmd_serialization(uint64_t version, const char * buf, size_t len) {
-        // copy buf to data
-        data.assign(buf, buf + len);
-        uint64_t ver_in = read<uint64_t>();
-        if (ver_in != version) {
-            throw std::runtime_error("version mismatch");
-        }
-        this->version = ver_in;
-    }
-
-    template <typename T>
-    void write(T value) {
-        static_assert(std::is_trivially_copyable<T>::value && !std::is_same<T, bool>::value,
-            "T must be trivially copyable and not bool");
-        const char * p = reinterpret_cast<const char *>(&value);
-        data.insert(data.end(), p, p + sizeof(T));
-    }
-
-    template <typename T>
-    T read() {
-        static_assert(std::is_trivially_copyable<T>::value && !std::is_same<T, bool>::value,
-            "T must be trivially copyable and not bool");
-        if (read_pos + sizeof(T) > data.size()) {
-            throw std::runtime_error("read OOB");
-        }
-        T value;
-        std::memcpy(&value, data.data() + read_pos, sizeof(T));
-        read_pos += sizeof(T);
-        return value;
-    }
-
-};
-
-template <>
-void mtmd_serialization::write<bool>(bool value) {
-    write<uint8_t>(value ? 1 : 0);
-}
-template <>
-bool mtmd_serialization::read<bool>() {
-    return read<uint8_t>() != 0;
-}
-
-template <>
-void mtmd_serialization::write<std::string>(std::string value) {
-    write<uint64_t>(value.size());
-    data.insert(data.end(), value.begin(), value.end());
-}
-template <>
-std::string mtmd_serialization::read<std::string>() {
-    uint64_t len = read<uint64_t>();
-    if (read_pos + len > data.size()) {
-        throw std::runtime_error("read_string OOB");
-    }
-    std::string str(data.data() + read_pos, len);
-    read_pos += len;
-    return str;
-}
-
-// only mtmd.cpp needs these, so they're implemented here rather than in clip-impl.h
-void clip_image_f32::serialize(mtmd_serialization & ser) const {
-    // remember to bump MTMD_SERIALIZATION_VERSION if this is changed
-    // note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder
-    ser.write(add_viewsep);
-    ser.write(add_newline);
-    ser.write((int32_t)nx_);
-    ser.write((int32_t)ny_);
-}
-void clip_image_f32::deserialize(mtmd_serialization & ser) {
-    add_viewsep = ser.read<bool>();
-    add_newline = ser.read<bool>();
-    nx_ = ser.read<int32_t>();
-    ny_ = ser.read<int32_t>();
-    buf.clear(); // always a placeholder after loading
-}
-
-void clip_image_f32_batch::serialize(mtmd_serialization & ser) const {
-    // remember to bump MTMD_SERIALIZATION_VERSION if this is changed
-    ser.write(is_audio);
-    ser.write<uint64_t>(entries.size());
-    for (const auto & entry : entries) {
-        entry.serialize(ser);
-    }
-}
-void clip_image_f32_batch::deserialize(mtmd_serialization & ser) {
-    is_audio = ser.read<bool>();
-    uint64_t n = ser.read<uint64_t>();
-    constexpr size_t min_entry_bytes = sizeof(uint8_t) * 2 + sizeof(int32_t) * 2;
-    if (n > (ser.data.size() - ser.read_pos) / min_entry_bytes) {
-        throw std::runtime_error("entries count exceeds buffer size");
-    }
-    entries.clear();
-    entries.reserve(n);
-    for (uint64_t i = 0; i < n; i++) {
-        clip_image_f32 entry;
-        entry.deserialize(ser);
-        entries.push_back(std::move(entry));
-    }
-}
-
 // for still image data, layout is RGBRGBRGB...
 // length of data must be nx * ny * 3 bytes
 //
@@ -198,7 +83,6 @@ enum mtmd_pos_type {
     MTMD_POS_TYPE_NORMAL,    // number of positions equals to number of tokens
     MTMD_POS_TYPE_MROPE,     // qwen-vl mrope style, each image takes max(t,h,w) position indexes
     MTMD_POS_TYPE_HUNYUANVL, // HunyuanVL mrope + BOI/EOI/newline layout with XD-RoPE dim-3
-    MTMD_POS_TYPE_COUNT,     // for validation
 };
 
 struct mtmd_image_tokens {
@@ -252,30 +136,6 @@ struct mtmd_image_tokens {
             id
         };
     }
-
-    void serialize(mtmd_serialization & ser) const {
-        // remember to bump MTMD_SERIALIZATION_VERSION if this is changed
-        ser.write(nx);
-        ser.write(ny);
-        ser.write((uint32_t)pos);
-        ser.write(image_idx);
-        ser.write(n_temporal_merge);
-        ser.write(id);
-        batch_f32.serialize(ser);
-    }
-    void deserialize(mtmd_serialization & ser) {
-        nx = ser.read<uint32_t>();
-        ny = ser.read<uint32_t>();
-        uint32_t pos_raw = ser.read<uint32_t>();
-        if (pos_raw >= MTMD_POS_TYPE_COUNT) {
-            throw std::runtime_error("invalid pos type");
-        }
-        pos = (mtmd_pos_type)pos_raw;
-        image_idx = ser.read<uint32_t>();
-        n_temporal_merge = ser.read<uint32_t>();
-        id = ser.read<std::string>();
-        batch_f32.deserialize(ser);
-    }
 };
 using mtmd_image_tokens_ptr = std::unique_ptr<mtmd_image_tokens>;
 
@@ -301,18 +161,6 @@ struct mtmd_audio_tokens {
             id
         };
     }
-
-    void serialize(mtmd_serialization & ser) const {
-        // remember to bump MTMD_SERIALIZATION_VERSION if this is changed
-        ser.write(n_tokens);
-        ser.write(id);
-        batch_f32.serialize(ser);
-    }
-    void deserialize(mtmd_serialization & ser) {
-        n_tokens = ser.read<uint32_t>();
-        id = ser.read<std::string>();
-        batch_f32.deserialize(ser);
-    }
 };
 using mtmd_audio_tokens_ptr = std::unique_ptr<mtmd_audio_tokens>;
 
@@ -344,66 +192,6 @@ struct mtmd_input_chunk {
         }
         return false;
     }
-
-    void serialize(mtmd_serialization & ser) const {
-        // remember to bump MTMD_SERIALIZATION_VERSION if this is changed
-        ser.write((uint32_t)type);
-
-        ser.write<uint64_t>(tokens_text.size());
-        for (llama_token tok : tokens_text) {
-            ser.write((int32_t)tok);
-        }
-
-        ser.write(tokens_image != nullptr);
-        if (tokens_image) {
-            tokens_image->serialize(ser);
-        }
-
-        ser.write(tokens_audio != nullptr);
-        if (tokens_audio) {
-            tokens_audio->serialize(ser);
-        }
-    }
-    void deserialize(mtmd_serialization & ser) {
-        uint32_t type_raw = ser.read<uint32_t>();
-        if (type_raw >= MTMD_INPUT_CHUNK_TYPE_COUNT) {
-            throw std::runtime_error("invalid chunk type");
-        }
-        type = (mtmd_input_chunk_type)type_raw;
-
-        uint64_t n_tokens_text = ser.read<uint64_t>();
-        // reject before resize() so a tiny corrupted/malicious buffer can't force a huge allocation
-        if (n_tokens_text > (ser.data.size() - ser.read_pos) / sizeof(int32_t)) {
-            throw std::runtime_error("tokens_text length exceeds buffer size");
-        }
-        tokens_text.resize(n_tokens_text);
-        for (uint64_t i = 0; i < n_tokens_text; i++) {
-            tokens_text[i] = (llama_token)ser.read<int32_t>();
-        }
-
-        if (ser.read<bool>()) {
-            tokens_image = std::make_unique<mtmd_image_tokens>();
-            tokens_image->deserialize(ser);
-        } else {
-            tokens_image.reset();
-        }
-
-        if (ser.read<bool>()) {
-            tokens_audio = std::make_unique<mtmd_audio_tokens>();
-            tokens_audio->deserialize(ser);
-        } else {
-            tokens_audio.reset();
-        }
-
-        // catch buffers where the declared type doesn't match which payload is actually present,
-        // so a mismatched chunk can't slip through and null-deref/abort later in an accessor
-        if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE && !tokens_image) {
-            throw std::runtime_error("type is IMAGE but tokens_image is missing");
-        }
-        if (type == MTMD_INPUT_CHUNK_TYPE_AUDIO && !tokens_audio) {
-            throw std::runtime_error("type is AUDIO but tokens_audio is missing");
-        }
-    }
 };
 
 struct mtmd_input_chunks {
@@ -2255,42 +2043,6 @@ void mtmd_input_chunk_free(mtmd_input_chunk * chunk) {
     }
 }
 
-int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len) {
-    try {
-        mtmd_serialization ser(MTMD_SERIALIZATION_VERSION);
-        chunk->serialize(ser);
-
-        if (expected_out_len) {
-            *expected_out_len = ser.data.size();
-        }
-        if (!out_buf) {
-            // caller is only querying the required size
-            return 0;
-        }
-        if (out_len < ser.data.size()) {
-            LOG_ERR("%s: out_buf is too small, need %zu bytes, got %zu\n", __func__, ser.data.size(), out_len);
-            return -1;
-        }
-        std::memcpy(out_buf, ser.data.data(), ser.data.size());
-        return 0;
-    } catch (const std::exception & e) {
-        LOG_ERR("%s: %s\n", __func__, e.what());
-        return -1;
-    }
-}
-
-mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len) {
-    try {
-        mtmd_serialization ser(MTMD_SERIALIZATION_VERSION, buf, len);
-        mtmd::input_chunk_ptr chunk(new mtmd_input_chunk());
-        chunk->deserialize(ser);
-        return chunk.release();
-    } catch (const std::exception & e) {
-        LOG_ERR("%s: %s\n", __func__, e.what());
-        return nullptr;
-    }
-}
-
 // mtmd_image_tokens
 
 size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens) {

tools/mtmd/mtmd.h

@@ -55,7 +55,6 @@ enum mtmd_input_chunk_type {
     MTMD_INPUT_CHUNK_TYPE_TEXT,
     MTMD_INPUT_CHUNK_TYPE_IMAGE,
     MTMD_INPUT_CHUNK_TYPE_AUDIO,
-    MTMD_INPUT_CHUNK_TYPE_COUNT, // for validation
 };
 
 // opaque types
@@ -233,15 +232,6 @@ MTMD_API llama_pos                  mtmd_input_chunk_get_n_pos       (const mtmd
 MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk);
 MTMD_API void               mtmd_input_chunk_free(mtmd_input_chunk * chunk);
 
-// save/load an input chunk to/from a buffer (useful for KV save/load)
-// important: only chunk's metadata will be saved, the actual image/audio data will not be saved
-// the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode()
-// out_buf can be nullptr (to query expected_out_len)
-// returns 0 on success, non-zero on failure
-MTMD_API int32_t            mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len);
-// returns nullptr on failure
-MTMD_API mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len);
-
 
 // mtmd_image_tokens
 //

tools/server/README.md

@@ -1076,10 +1076,6 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin
 | `llamacpp:n_tokens_max` | Counter | High watermark of the context size observed. |
 | `llamacpp:n_decode_total` | Counter | Total Number of llama_decode() calls. |
 | `llamacpp:n_busy_slots_per_decode` | Gauge | Average number of busy slots per llama_decode() call. |
-| `llamacpp:spec_decode_num_draft_tokens_total` | Counter | Total draft tokens generated (0 when spec-decode is off). |
-| `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). |
-| `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). |
-| `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). |
 
 ### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file.
 

tools/server/server-context.cpp

@@ -845,11 +845,6 @@ struct server_metrics {
     uint64_t n_decode_total     = 0;
     uint64_t n_busy_slots_total = 0;
 
-    uint64_t n_draft_tokens_total      = 0;
-    uint64_t n_draft_accepted_total    = 0;
-    uint64_t n_draft_verif_steps_total = 0;
-    std::vector<uint64_t> n_accepted_per_pos_total;
-
     void init() {
         t_start = ggml_time_us();
     }
@@ -868,17 +863,6 @@ struct server_metrics {
         n_tokens_predicted         += slot.n_decoded;
         t_tokens_generation        += slot.t_token_generation;
         t_tokens_generation_total  += slot.t_token_generation;
-
-        n_draft_tokens_total      += slot.n_draft_total;
-        n_draft_accepted_total    += slot.n_draft_accepted;
-        n_draft_verif_steps_total += slot.n_draft_verif_steps;
-
-        if (n_accepted_per_pos_total.size() < slot.n_accepted_per_pos.size()) {
-            n_accepted_per_pos_total.resize(slot.n_accepted_per_pos.size(), 0);
-        }
-        for (size_t i = 0; i < slot.n_accepted_per_pos.size(); i++) {
-            n_accepted_per_pos_total[i] += slot.n_accepted_per_pos[i];
-        }
     }
 
     void on_decoded(const std::vector<server_slot> & slots) {
@@ -2568,11 +2552,6 @@ private:
                     res->n_decode_total          = metrics.n_decode_total;
                     res->n_busy_slots_total      = metrics.n_busy_slots_total;
 
-                    res->n_draft_tokens_total      = metrics.n_draft_tokens_total;
-                    res->n_draft_accepted_total    = metrics.n_draft_accepted_total;
-                    res->n_draft_verif_steps_total = metrics.n_draft_verif_steps_total;
-                    res->n_accepted_per_pos_total  = metrics.n_accepted_per_pos_total;
-
                     if (task.metrics_reset_bucket) {
                         metrics.reset_bucket();
                     }
@@ -4461,18 +4440,6 @@ void server_routes::init_routes() {
                     {"name",  "n_tokens_max"},
                     {"help",  "Largest observed n_tokens."},
                     {"value",  res_task->n_tokens_max}
-            }, {
-                    {"name",  "spec_decode_num_draft_tokens_total"},
-                    {"help",  "Total draft tokens generated"},
-                    {"value",  res_task->n_draft_tokens_total}
-            }, {
-                    {"name",  "spec_decode_num_accepted_tokens_total"},
-                    {"help",  "Total draft tokens accepted by the target model"},
-                    {"value",  res_task->n_draft_accepted_total}
-            }, {
-                    {"name",  "spec_decode_num_drafts_total"},
-                    {"help",  "Total speculative decoding verification steps"},
-                    {"value",  res_task->n_draft_verif_steps_total}
             }}},
             {"gauge", {{
                     {"name",  "prompt_tokens_seconds"},
@@ -4514,17 +4481,6 @@ void server_routes::init_routes() {
             }
         }
 
-        // labeled counter: one time series per draft position
-        if (!res_task->n_accepted_per_pos_total.empty()) {
-            prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
-                          " Accepted tokens per draft position\n"
-                       << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
-            for (size_t i = 0; i < res_task->n_accepted_per_pos_total.size(); i++) {
-                prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
-                           << i << "\"} " << res_task->n_accepted_per_pos_total[i] << "\n";
-            }
-        }
-
         res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start);
         res->content_type = "text/plain; version=0.0.4";
         res->status = 200;

tools/server/server-models.cpp

@@ -70,188 +70,6 @@ struct server_subproc {
     }
 };
 
-struct server_lru_sched {
-    server_lru_sched(server_models & models) : models(models) {}
-
-    bool has_capacity(std::unique_lock<std::mutex> & lk) {
-        check_lock(lk);
-        return models.base_params.models_max <= 0
-            || count_running() < (size_t) models.base_params.models_max;
-    }
-
-    // returns "" if no model can be given up
-    std::string pick_victim(std::unique_lock<std::mutex> & lk, const std::string & exclude) {
-        check_lock(lk);
-        std::string victim;
-        int64_t victim_last_used = 0;
-        for (const auto & m : models.mapping) {
-            if (m.first == exclude) {
-                continue;
-            }
-            // a busy model is mid-request, one still coming up has no request to finish
-            if (m.second.req_count != 0 || !m.second.meta.is_ready_or_sleep()) {
-                continue;
-            }
-            if (victim.empty() || m.second.meta.last_used < victim_last_used) {
-                victim           = m.first;
-                victim_last_used = m.second.meta.last_used;
-            }
-        }
-        return victim;
-    }
-
-    // requests wanting the same model share one entry, so they all need only one slot
-    // and all get unblocked by the single load that entry performs
-    void join(std::unique_lock<std::mutex> & lk, const std::string & model_id) {
-        check_lock(lk);
-        if (entry_t * e = find(model_id)) {
-            e->n_waiters++;
-            SRV_INF("request for name=%s joined the queue, %d waiting\n", model_id.c_str(), e->n_waiters);
-            return;
-        }
-        queue.push_back({ model_id, 1, false, false });
-        SRV_INF("models_max reached, request for name=%s queued at position %zu\n",
-                model_id.c_str(), queue.size());
-    }
-
-    void leave(std::unique_lock<std::mutex> & lk, const std::string & model_id) {
-        check_lock(lk);
-        for (auto it = queue.begin(); it != queue.end(); ++it) {
-            if (it->model_id == model_id) {
-                if (--it->n_waiters <= 0) {
-                    queue.erase(it); // last one waiting for this model went away
-                }
-                return;
-            }
-        }
-    }
-
-    bool queue_empty(std::unique_lock<std::mutex> & lk) {
-        check_lock(lk);
-        return queue.empty();
-    }
-
-    // true if it is this model's turn to load, and nobody is loading it yet
-    bool try_claim(std::unique_lock<std::mutex> & lk, const std::string & model_id) {
-        check_lock(lk);
-        if (queue.empty() || queue.front().model_id != model_id || queue.front().loading) {
-            return false;
-        }
-        if (!has_capacity(lk)) {
-            return false;
-        }
-        queue.front().loading = true;
-        return true;
-    }
-
-    // ok means the model is up: drop the entry, the other waiters just watch its status now
-    void claim_done(std::unique_lock<std::mutex> & lk, const std::string & model_id, bool ok) {
-        check_lock(lk);
-        for (auto it = queue.begin(); it != queue.end(); ++it) {
-            if (it->model_id == model_id) {
-                if (ok) {
-                    queue.erase(it);
-                } else {
-                    it->loading = false;
-                }
-                return;
-            }
-        }
-    }
-
-    // a model is on its way out for this entry, so other requests do not also give up one
-    void mark_slot_pending(std::unique_lock<std::mutex> & lk, const std::string & model_id) {
-        check_lock(lk);
-        if (entry_t * e = find(model_id)) {
-            e->slot_pending = true;
-        }
-    }
-
-    // model_id went idle: give up its slot if a queued request needs one
-    // thread-safe, caller must NOT hold models.mutex
-    void on_model_idle(const std::string & model_id) {
-        if (models.base_params.models_max <= 0) {
-            return; // no limit, nothing is ever queued
-        }
-        {
-            std::unique_lock<std::mutex> lk(models.mutex);
-            if (queue.empty()) {
-                return;
-            }
-            size_t promised     = 0;
-            bool   has_unserved = false;
-            for (const auto & e : queue) {
-                if (e.needs_slot()) {
-                    has_unserved = true;
-                } else {
-                    promised++;
-                }
-            }
-            if (!has_unserved) {
-                return;
-            }
-            if ((int) count_running() - (int) promised < models.base_params.models_max) {
-                return; // a slot is already on its way
-            }
-            // never give up a model that a queued request wants
-            for (const auto & e : queue) {
-                if (e.model_id == model_id) {
-                    return;
-                }
-            }
-            auto it = models.mapping.find(model_id);
-            if (it == models.mapping.end() || it->second.req_count != 0 || !it->second.meta.is_ready_or_sleep()) {
-                return;
-            }
-            for (auto & e : queue) {
-                if (!e.slot_pending) {
-                    e.slot_pending = true;
-                    break;
-                }
-            }
-        }
-        SRV_INF("model name=%s went idle, giving up its slot to a queued request\n", model_id.c_str());
-        models.unload(model_id);
-    }
-
-  private:
-    struct entry_t {
-        std::string model_id;
-        int  n_waiters;    // requests waiting for this model
-        bool slot_pending; // a model is already being evicted for this entry
-        bool loading;      // one of the waiters is doing the load right now
-
-        // a slot is already coming, or already taken by the load in flight
-        bool needs_slot() const { return !slot_pending && !loading; }
-    };
-
-    entry_t * find(const std::string & model_id) {
-        for (auto & e : queue) {
-            if (e.model_id == model_id) {
-                return &e;
-            }
-        }
-        return nullptr;
-    }
-
-    void check_lock(std::unique_lock<std::mutex> & lk) {
-        GGML_ASSERT(lk.owns_lock() && lk.mutex() == &models.mutex);
-    }
-
-    size_t count_running() {
-        size_t count = 0;
-        for (const auto & m : models.mapping) {
-            if (m.second.meta.is_running()) {
-                count++;
-            }
-        }
-        return count;
-    }
-
-    server_models & models;
-    std::deque<entry_t> queue;
-};
-
 // short loopback budget for the resumable stream router to child JSON calls (probe, lookup,
 // delete). distinct from params.timeout_read/write which only applies to the generation proxy
 static constexpr int STREAM_LOOKUP_TIMEOUT_MS = 250;
@@ -411,8 +229,7 @@ server_models::server_models(
             : ctx_preset(LLAMA_EXAMPLE_SERVER),
               base_params(params),
               base_env(get_environment()),
-              base_preset(ctx_preset.load_from_args(argc, argv)),
-              sched(std::make_unique<server_lru_sched>(*this)) {
+              base_preset(ctx_preset.load_from_args(argc, argv)) {
     // clean up base preset
     unset_reserved_args(base_preset, true);
     // set binary path
@@ -424,11 +241,8 @@ server_models::server_models(
         LOG_WRN("using original argv[0] as fallback: %s\n", argv[0]);
     }
     load_models();
-    debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty();
 }
 
-server_models::~server_models() = default;
-
 void server_models::add_model(server_model_meta && meta) {
     if (mapping.find(meta.name) != mapping.end()) {
         throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str()));
@@ -899,15 +713,22 @@ void server_models::unload_lru() {
         return; // no limit
     }
     // remove one of the servers if we passed the models_max (least recently used - LRU)
-    std::string lru_model_name;
+    std::string lru_model_name = "";
+    int64_t lru_last_used = ggml_time_ms();
+    size_t count_active = 0;
     {
         std::unique_lock<std::mutex> lk(mutex);
-        if (sched->has_capacity(lk)) {
-            return;
+        for (const auto & m : mapping) {
+            if (m.second.meta.is_running()) {
+                count_active++;
+                if (m.second.meta.last_used < lru_last_used) {
+                    lru_model_name = m.first;
+                    lru_last_used = m.second.meta.last_used;
+                }
+            }
         }
-        lru_model_name = sched->pick_victim(lk, "");
     }
-    if (!lru_model_name.empty()) {
+    if (!lru_model_name.empty() && count_active >= (size_t)base_params.models_max) {
         SRV_INF("models_max limit reached, removing LRU name=%s\n", lru_model_name.c_str());
         unload(lru_model_name);
         // wait for unload to complete
@@ -925,11 +746,6 @@ void server_models::load(const std::string & name) {
 }
 
 void server_models::load(const std::string & name, const load_options & opts) {
-    if (debug_fake_timing) {
-        // do not hold the mutex here, other requests must keep making progress
-        std::this_thread::sleep_for(std::chrono::seconds(2));
-    }
-
     if (!opts.custom_meta.has_value()) {
         if (!has_model(name)) {
             throw std::runtime_error("model name=" + name + " is not found");
@@ -1322,7 +1138,7 @@ void server_models::wait(std::unique_lock<std::mutex> & lk, const std::string &
     });
 }
 
-bool server_models::ensure_model_ready(const std::string & name, const std::function<bool()> & should_stop) {
+bool server_models::ensure_model_ready(const std::string & name) {
     auto meta = get_meta(name);
     if (!meta.has_value()) {
         throw std::runtime_error("model name=" + name + " is not found");
@@ -1333,112 +1149,25 @@ bool server_models::ensure_model_ready(const std::string & name, const std::func
     if (meta->status == SERVER_MODEL_STATUS_SLEEPING) {
         return false; // child is sleeping but still running; new request will wake it up
     }
-
-    bool queued   = false;
-    bool did_load = false;
-    std::string victim;
-    {
-        std::unique_lock<std::mutex> lk(mutex);
-        auto it = mapping.find(name);
-        if (it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_UNLOADED) {
-            bool has_capacity = sched->has_capacity(lk);
-            if (has_capacity && sched->queue_empty(lk)) {
-                lk.unlock();
-                SRV_INF("model name=%s is not loaded, loading...\n", name.c_str());
-                load(name);
-                did_load = true;
-            } else {
-                // also queue when a slot looks free but others wait already, else they starve
-                sched->join(lk, name);
-                queued = true;
-                if (!has_capacity) {
-                    // an idle model may sit here right now, do not wait for a request to end
-                    victim = sched->pick_victim(lk, name);
-                    if (!victim.empty()) {
-                        sched->mark_slot_pending(lk, name);
-                    }
-                }
-            }
-        }
-    }
-    if (!victim.empty()) {
-        SRV_INF("evicting idle LRU name=%s to make room for name=%s\n", victim.c_str(), name.c_str());
-        unload(victim);
+    if (meta->status == SERVER_MODEL_STATUS_UNLOADED) {
+        SRV_INF("model name=%s is not loaded, loading...\n", name.c_str());
+        load(name);
     }
 
-    // while queued, this is also where the load happens: the head of the queue does it
+    // wait for loading to complete
     SRV_INF("waiting until model name=%s is fully loaded...\n", name.c_str());
-    std::unique_lock<std::mutex> lk(mutex);
-    auto leave_queue = [this, &queued, &lk, &name]() {
-        if (queued) {
-            sched->leave(lk, name);
-            queued = false;
+    wait(name, [&meta](const server_model_meta & new_meta) {
+        if (new_meta.status != SERVER_MODEL_STATUS_LOADING) {
+            meta = new_meta; // update meta for final check after wait
+            return true;
         }
-    };
+        return false;
+    });
 
-    try {
-        bool saw_loading = false;
-        while (true) {
-            auto it = mapping.find(name);
-            if (it == mapping.end()) {
-                break; // removed by another code path, nothing to wait for
-            }
-            const server_model_status status = it->second.meta.status;
-
-            if (status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_SLEEPING) {
-                break;
-            }
-            if (status == SERVER_MODEL_STATUS_DOWNLOADING || status == SERVER_MODEL_STATUS_DOWNLOADED) {
-                break; // do not wait on a download child
-            }
-            if (status == SERVER_MODEL_STATUS_LOADING) {
-                saw_loading = true;
-            } else if (status == SERVER_MODEL_STATUS_UNLOADED) {
-                if (did_load || saw_loading) {
-                    // a spawn happened and the instance came back down
-                    if (it->second.meta.is_failed()) {
-                        throw std::runtime_error("model name=" + name + " failed to load");
-                    }
-                    break; // unloaded by another code path, caller reports "not running"
-                }
-                if (!queued) {
-                    break; // not queued, and the load someone else started fell over
-                }
-            }
-
-            if (should_stop && should_stop()) {
-                // if a model was evicted for us, the free slot goes to the next waiter
-                throw std::runtime_error("request cancelled while waiting for model name=" + name);
-            }
-
-            // our turn: our model is at the head, and a slot really did free up
-            if (status == SERVER_MODEL_STATUS_UNLOADED && sched->try_claim(lk, name)) {
-                lk.unlock();
-                bool ok = true;
-                try {
-                    SRV_INF("slot available, loading queued model name=%s\n", name.c_str());
-                    load(name);
-                    did_load = true;
-                } catch (const std::exception & e) {
-                    // lost a race for the slot, stay in line and retry
-                    SRV_WRN("queued load of name=%s did not go through: %s\n", name.c_str(), e.what());
-                    ok = false;
-                }
-                lk.lock();
-                sched->claim_done(lk, name, ok);
-                if (ok) {
-                    queued = false; // entry is gone, the other waiters watch the status now
-                }
-                continue;
-            }
-
-            cv.wait_for(lk, std::chrono::milliseconds(200));
-        }
-    } catch (...) {
-        leave_queue();
-        throw;
+    // check final status
+    if (!meta.has_value() || meta->is_failed()) {
+        throw std::runtime_error("model name=" + name + " failed to load");
     }
-    leave_queue();
 
     return true;
 }
@@ -1451,16 +1180,9 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
     if (!meta->is_running()) {
         throw std::invalid_argument("model name=" + name + " is not running");
     }
-    {
+    if (update_last_used) {
         std::unique_lock<std::mutex> lk(mutex);
-        if (update_last_used) {
-            mapping[name].meta.last_used = ggml_time_ms();
-        }
-        mapping[name].req_count++;
-    }
-    if (debug_fake_timing) {
-        // sleep after req_count++, so the model counts as busy while we wait here
-        std::this_thread::sleep_for(std::chrono::seconds(2));
+        mapping[name].meta.last_used = ggml_time_ms();
     }
     SRV_INF("proxying request to model %s on port %d\n", name.c_str(), meta->port);
     std::string proxy_path = req.path;
@@ -1476,29 +1198,13 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
             req.headers,
             req.body,
             req.files,
-            // a detached request belongs to a replay session
-            detached
-                ? std::function<bool()>([]() { return false; })
-                : req.should_stop,
+            // a detached request belongs to a replay session that outlives the client socket:
+            // it reaches the child even when the downstream died during the load wait, the
+            // session buffer is the recipient and DELETE remains the stop
+            detached ? std::function<bool()>([]() { return false; }) : req.should_stop,
             base_params.timeout_read,
             base_params.timeout_write
             );
-
-    proxy->cleanup = [this, name]() {
-        bool went_idle = false;
-        {
-            std::unique_lock<std::mutex> lk(mutex);
-            auto it = mapping.find(name);
-            if (it != mapping.end() && it->second.req_count > 0) {
-                it->second.req_count--;
-                went_idle = it->second.req_count == 0;
-            }
-        }
-        if (went_idle) {
-            sched->on_model_idle(name);
-        }
-    };
-
     return proxy;
 }
 
@@ -1862,7 +1568,7 @@ void server_models_routes::init_routes() {
             return error_res;
         }
         if (autoload) {
-            models.ensure_model_ready(name, req.should_stop);
+            models.ensure_model_ready(name);
         }
         return models.proxy_request(req, method, name, false);
     };
@@ -1882,9 +1588,7 @@ void server_models_routes::init_routes() {
         // this request instead of leaving an orphan generation
         std::string conv_id = server_stream_conv_id_from_headers(req.headers);
         uint64_t ticket = models.conv_models.remember(conv_id, name);
-        // a dead socket must not cancel a session request, only a stop does (checked right below)
-        auto should_stop = ticket == 0 ? req.should_stop : nullptr;
-        bool waited = autoload && models.ensure_model_ready(name, should_stop);
+        bool waited = autoload && models.ensure_model_ready(name);
         if (ticket != 0 && !models.conv_models.alive(conv_id, ticket)) {
             SRV_INF("request for conv_id=%s cancelled while model name=%s was loading\n",
                     conv_id.c_str(), name.c_str());
@@ -2360,7 +2064,7 @@ server_http_proxy::server_http_proxy(
     cli->set_write_timeout(timeout_read, 0); // reversed for cli (client) vs srv (server)
     cli->set_read_timeout(timeout_write, 0);
     this->status = 500; // to be overwritten upon response
-    this->cleanup_pipes = [pipe]() {
+    this->cleanup = [pipe]() {
         pipe->close_read();
         pipe->close_write();
     };
@@ -2375,8 +2079,9 @@ server_http_proxy::server_http_proxy(
         return has_next; // false if EOF or pipe broken
     };
 
-    // build the header message forwarded to the reader thread, stripping internal proxy headers
-    auto make_header_msg = [](const httplib::Response & response) {
+    // wire up the HTTP client
+    // note: do NOT capture `this` pointer, as it may be destroyed before the thread ends
+    httplib::ResponseHandler response_handler = [pipe, cli](const httplib::Response & response) {
         msg_t msg;
         msg.status = response.status;
         for (const auto & [key, value] : response.headers) {
@@ -2390,17 +2095,7 @@ server_http_proxy::server_http_proxy(
             }
             msg.headers[key] = value;
         }
-        return msg;
-    };
-
-    // true once response_handler has already forwarded the headers
-    auto headers_sent = std::make_shared<std::atomic<bool>>(false);
-
-    // wire up the HTTP client
-    // note: do NOT capture `this` pointer, as it may be destroyed before the thread ends
-    httplib::ResponseHandler response_handler = [pipe, headers_sent, make_header_msg](const httplib::Response & response) {
-        headers_sent->store(true);
-        return pipe->write(make_header_msg(response)); // send headers first
+        return pipe->write(std::move(msg)); // send headers first
     };
     httplib::ContentReceiverWithProgress content_receiver = [pipe](const char * data, size_t data_length, size_t, size_t) {
         // send data chunks
@@ -2474,16 +2169,13 @@ server_http_proxy::server_http_proxy(
 
     // start the proxy thread
     SRV_DBG("start proxy thread %s %s\n", req.method.c_str(), req.path.c_str());
-    this->thread = std::thread([cli, pipe, req, headers_sent, make_header_msg]() {
+    this->thread = std::thread([cli, pipe, req]() {
         auto result = cli->send(std::move(req));
         if (result.error() != httplib::Error::Success) {
             auto err_str = httplib::to_string(result.error());
             SRV_ERR("http client error: %s\n", err_str.c_str());
             pipe->write({{}, 500, "", ""}); // header
             pipe->write({{}, 0, "proxy error: " + err_str, ""}); // body
-        } else if (!headers_sent->load()) {
-            // httplib skips response_handler for bodyless statuses like 204, send headers here instead
-            pipe->write(make_header_msg(*result));
         }
         pipe->close_write(); // signal EOF to reader
         SRV_DBG("%s", "client request thread ended\n");

tools/server/server-models.h

@@ -84,6 +84,7 @@ struct server_model_meta {
     int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
     int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
     mtmd_caps multimodal; // multimodal capabilities
+    // bool need_download = false; // whether the model needs to be downloaded before loading // TODO @ngxson: implement this
 
     bool is_ready() const {
         return status == SERVER_MODEL_STATUS_LOADED;
@@ -93,10 +94,6 @@ struct server_model_meta {
         return status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_LOADING || status == SERVER_MODEL_STATUS_SLEEPING;
     }
 
-    bool is_ready_or_sleep() const {
-        return status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_SLEEPING;
-    }
-
     bool is_failed() const {
         return status == SERVER_MODEL_STATUS_UNLOADED && exit_code != 0;
     }
@@ -106,19 +103,16 @@ struct server_model_meta {
 };
 
 struct server_models_routes;
-struct server_subproc;   // defined in server-models.cpp
-struct server_lru_sched; // defined in server-models.cpp
+struct server_subproc; // defined in server-models.cpp
 
 struct server_models {
     friend struct server_models_routes;
-    friend struct server_lru_sched;
 
 private:
     struct instance_t {
         std::shared_ptr<server_subproc> subproc; // shared between main thread and monitoring thread
         std::thread th;
         server_model_meta meta;
-        int req_count = 0; // number of active proxy requests
     };
 
     std::mutex mutex;
@@ -197,12 +191,6 @@ private:
     std::vector<std::string> base_env;
     common_preset base_preset; // base preset from llama-server CLI args
 
-    // queue of requests waiting for a models_max slot
-    std::unique_ptr<server_lru_sched> sched;
-
-    // if true, add some delay to simulate works (useful for testing)
-    bool debug_fake_timing = false;
-
     void update_meta(const std::string & name, const server_model_meta & meta);
 
     // unload least recently used models if the limit is reached
@@ -219,7 +207,6 @@ public:
     conv_model_tracker conv_models;
 
     server_models(const common_params & params, int argc, char ** argv);
-    ~server_models();
 
     server_response sse; // for real-time updates via SSE endpoint
 
@@ -276,9 +263,7 @@ public:
     // ensure the model is in ready state (thread-safe)
     // return false if model is ready
     // otherwise, load the model and blocking wait until it's ready, then return true (meta may need to be refreshed)
-    // if models_max is reached, the request waits in a queue until a slot frees up
-    // throws if the load fails, or if should_stop fires while waiting
-    bool ensure_model_ready(const std::string & name, const std::function<bool()> & should_stop = nullptr);
+    bool ensure_model_ready(const std::string & name);
 
     // proxy an HTTP request to the model instance
     server_http_res_ptr proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used, bool detached = false);
@@ -358,6 +343,7 @@ struct server_models_routes {
  */
 struct server_http_proxy : server_http_res {
     std::function<void()> cleanup = nullptr;
+public:
     server_http_proxy(const std::string & method,
                       const std::string & scheme,
                       const std::string & host,
@@ -371,15 +357,11 @@ struct server_http_proxy : server_http_res {
                       int32_t timeout_write
                       );
     ~server_http_proxy() {
-        if (cleanup_pipes) {
-            cleanup_pipes();
-        }
         if (cleanup) {
             cleanup();
         }
     }
 private:
-    std::function<void()> cleanup_pipes = nullptr;
     std::thread thread;
     struct msg_t {
         std::map<std::string, std::string> headers;

tools/server/server-schema.cpp

@@ -519,7 +519,7 @@ task_params eval_llama_cmpl_schema(
                 const json & data) {
     task_params params;
 
-    // Sampling parameter defaults are loaded from the global server context (but individual requests can still override them)
+    // Sampling parameter defaults are loaded from the global server context (but individual requests can still them)
     params.sampling      = params_base.sampling;
     params.speculative   = params_base.speculative;
     params.n_keep        = params_base.n_keep;

tools/server/server-task.cpp

@@ -1560,11 +1560,6 @@ json server_task_result_metrics::to_json() {
         { "n_decode_total",                  n_decode_total },
         { "n_busy_slots_total",              n_busy_slots_total },
 
-        { "n_draft_tokens_total",            n_draft_tokens_total },
-        { "n_draft_accepted_total",          n_draft_accepted_total },
-        { "n_draft_verif_steps_total",       n_draft_verif_steps_total },
-        { "n_accepted_per_pos_total",        n_accepted_per_pos_total },
-
         { "slots",                           slots_data },
     };
 }

tools/server/server-task.h

@@ -532,11 +532,6 @@ struct server_task_result_metrics : server_task_result {
     uint64_t n_decode_total     = 0;
     uint64_t n_busy_slots_total = 0;
 
-    uint64_t n_draft_tokens_total      = 0;
-    uint64_t n_draft_accepted_total    = 0;
-    uint64_t n_draft_verif_steps_total = 0;
-    std::vector<uint64_t> n_accepted_per_pos_total;
-
     // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
     // therefore, we use json to temporarily store the slot.to_json() result
     json slots_data = json::array();

tools/server/server-tools.cpp

@@ -30,14 +30,20 @@ namespace fs = std::filesystem;
 // internal helpers
 //
 
-// a child process writes in the OEM code page, so accented output would reach
-// the JSON layer as invalid bytes. run() spawns without a console, so the
-// console code page never applies
-static std::string console_output_to_utf8(const std::string & text) {
 #if defined(_WIN32)
-    // a chunk can end mid sequence, so the incomplete tail is dropped first
-    if (text.empty() || is_valid_utf8(text.substr(0, validate_utf8(text)))) {
-        // never decode twice a child that already emits UTF-8
+// A chunk can end in the middle of a multi-byte sequence, so the incomplete
+// tail is dropped before validating what precedes it.
+static bool is_utf8_text(const std::string & text) {
+    return is_valid_utf8(text.substr(0, validate_utf8(text)));
+}
+
+// A child process writes its output in the OEM code page, which is not UTF-8
+// on a western Windows install, so accented text reaches the JSON layer as
+// invalid bytes and is replaced there. Text that already decodes as UTF-8 is
+// returned untouched, so a child that emits UTF-8 is never decoded twice.
+// run() spawns without a console, so the console code page does not apply.
+static std::string console_output_to_utf8(const std::string & text) {
+    if (text.empty() || is_utf8_text(text)) {
         return text;
     }
 
@@ -58,10 +64,12 @@ static std::string console_output_to_utf8(const std::string & text) {
     std::string utf8(utf8_len, '\0');
     WideCharToMultiByte(CP_UTF8, 0, wide.data(), wide_len, utf8.data(), utf8_len, nullptr, nullptr);
     return utf8;
-#else
-    return text;
-#endif
 }
+#else
+static std::string console_output_to_utf8(const std::string & text) {
+    return text;
+}
+#endif
 
 json server_tool::to_json() const {
     return {
@@ -86,30 +94,14 @@ enum class list_kind {
     all,   // both
 };
 
-// a narrow path uses the active code page on Windows, so every crossing between
-// a std::string (always UTF-8 here) and fs::path is converted explicitly
-static fs::path path_from_utf8(const std::string & s) {
-    return fs::u8path(s);
-}
-
-// '/' separators on every platform: Windows accepts them, the web UI needs them
-static std::string path_to_utf8(const fs::path & p) {
-    const auto s = p.generic_u8string();
-    return std::string(s.begin(), s.end());
-}
-
 // home directory, read once at first use (getenv is not thread safe against setenv)
 static const std::string & home_dir() {
     static const std::string home = [] {
-#ifdef _WIN32
-        // the narrow getenv would return the profile path in the active code page
-        const wchar_t * w = _wgetenv(L"HOME");
-        if (w == nullptr) w = _wgetenv(L"USERPROFILE");
-        return w ? path_to_utf8(fs::path(w)) : std::string();
-#else
         const char * h = getenv("HOME");
-        return h ? std::string(h) : std::string();
+#ifdef _WIN32
+        if (h == nullptr) h = getenv("USERPROFILE");
 #endif
+        return h ? std::string(h) : std::string();
     }();
     return home;
 }
@@ -148,14 +140,11 @@ public:
         std::string rel; // '/'-separated, relative to `base`
         bool is_dir = false;
     };
-    struct list_result {
-        std::vector<list_entry> entries;
-        std::string err;        // set when `base` is not a directory
-        bool truncated = false; // set when the walk could not see everything
-    };
-    // entries relative to `base`, which must already be resolved (absolute)
+    // entries relative to `base`; sets `err` if `base` isn't a directory
     // max_depth == 0 means unlimited, 1 means direct children of `base` only
-    virtual list_result list_entries(const std::string & base, int max_depth, list_kind kind) const = 0;
+    // `base` must already be resolved (absolute); `caller_path` is the path the
+    // caller passed, used only for error messages
+    virtual std::vector<list_entry> list_entries(const std::string & base, const std::string & caller_path, int max_depth, list_kind kind, std::string & err, bool & truncated) const = 0;
     // on_chunk, if set, is called with each chunk of output as it is read (before truncation cuts in);
     // returning false terminates the process early (e.g. the client disconnected)
     virtual exec_result run(
@@ -173,47 +162,37 @@ public:
     // expands a leading `~`, then resolves `path` against `cwd` (or the server
     // working directory when `cwd` is unset); the result is always absolute
     std::string resolve(const std::string & path) const override {
-        const std::string p = expand_home(path);
-
-        fs::path full = path_from_utf8(p);
-        if (!full.is_absolute()) {
-            if (cwd.empty()) {
-                std::error_code ec;
-                const fs::path cur = fs::current_path(ec);
-                if (ec) return p;
-                full = cur / full;
-            } else {
-                full = path_from_utf8(cwd) / full;
-            }
+        std::string p = expand_home(path);
+        if (fs::path(p).is_absolute()) {
+            return p;
         }
-
-        // drop "." and ".." so they never reach git or the client
-        full = full.lexically_normal();
-        // a trailing ".." normalizes to a path that ends with a separator
-        if (!full.has_filename() && full != full.root_path()) {
-            full = full.parent_path();
+        if (cwd.empty()) {
+            std::error_code ec;
+            fs::path cur = fs::current_path(ec);
+            if (ec) return p;
+            return (cur / p).string();
         }
-        return path_to_utf8(full);
+        return (fs::path(cwd) / p).string();
     }
 
     bool is_directory(const std::string & path) const override {
         std::error_code ec;
-        return fs::is_directory(path_from_utf8(resolve(path)), ec) && !ec;
+        return fs::is_directory(resolve(path), ec) && !ec;
     }
 
     bool is_regular_file(const std::string & path) const override {
         std::error_code ec;
-        return fs::is_regular_file(path_from_utf8(resolve(path)), ec) && !ec;
+        return fs::is_regular_file(resolve(path), ec) && !ec;
     }
 
     bool file_size(const std::string & path, uintmax_t & out_size) const override {
         std::error_code ec;
-        out_size = fs::file_size(path_from_utf8(resolve(path)), ec);
+        out_size = fs::file_size(resolve(path), ec);
         return !ec;
     }
 
     bool read_file(const std::string & path, std::string & out) const override {
-        std::ifstream f(path_from_utf8(resolve(path)), std::ios::binary);
+        std::ifstream f(resolve(path), std::ios::binary);
         if (!f) return false;
         std::ostringstream ss;
         ss << f.rdbuf();
@@ -223,7 +202,7 @@ public:
 
     bool write_file(const std::string & path, const std::string & content) const override {
         std::error_code ec;
-        fs::path fpath = path_from_utf8(resolve(path));
+        fs::path fpath(resolve(path));
         if (fpath.has_parent_path()) {
             fs::create_directories(fpath.parent_path(), ec);
             if (ec) return false;
@@ -234,13 +213,13 @@ public:
         return (bool) f;
     }
 
-    list_result list_entries(const std::string & base, int max_depth, list_kind kind) const override {
-        list_result out;
-
+    std::vector<list_entry> list_entries(const std::string & base, const std::string & caller_path, int max_depth, list_kind kind, std::string & err, bool & truncated) const override {
+        err.clear();
+        truncated = false;
         std::error_code ec;
         if (!fs::is_directory(base, ec) || ec) {
-            out.err = "path does not exist or is not a directory";
-            return out;
+            err = "path does not exist or is not a directory: " + caller_path;
+            return {};
         }
 
         const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(SERVER_TOOL_LIST_ENTRIES_TIMEOUT);
@@ -252,6 +231,7 @@ public:
                 SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, SERVER_TOOL_LIST_ENTRIES_TIMEOUT);
 
             if (res.exit_code == 0 && !res.timed_out) {
+                std::vector<list_entry> result;
                 std::istringstream iss(res.output);
                 std::string line;
                 while (std::getline(iss, line)) {
@@ -259,16 +239,15 @@ public:
                     if (line.empty()) continue;
                     std::replace(line.begin(), line.end(), '\\', '/');
                     if (max_depth > 0 && entry_depth(line) > max_depth) continue;
-                    if (is_regular_file(path_to_utf8(path_from_utf8(base) / path_from_utf8(line)))) {
-                        out.entries.push_back({line, false});
+                    if (is_regular_file((fs::path(base) / line).string())) {
+                        result.push_back({line, false});
                     }
                 }
-                return out;
+                return result;
             }
         }
 
-        out.entries = list_entries_fallback(base, max_depth, kind, deadline, out.truncated);
-        return out;
+        return list_entries_fallback(base, max_depth, kind, deadline, truncated);
     }
 
     exec_result run(
@@ -347,42 +326,6 @@ public:
 private:
     std::string cwd;
 
-    // a link can point back to an ancestor and loop forever, so it is never walked
-    static bool is_link(const fs::directory_entry & entry) {
-        std::error_code ec;
-        if (entry.is_symlink(ec) || ec) {
-            return true;
-        }
-#if defined(_WIN32)
-        // a junction looks like a plain directory to std::filesystem, so read the reparse tag
-        WIN32_FIND_DATAW data;
-        const HANDLE h = FindFirstFileW(entry.path().c_str(), &data);
-        if (h == INVALID_HANDLE_VALUE) {
-            return false;
-        }
-        FindClose(h);
-        if ((data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0) {
-            return false;
-        }
-        // other reparse points (cloud placeholder, dedup stub) are real directories
-        return data.dwReserved0 == IO_REPARSE_TAG_SYMLINK || data.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT;
-#else
-        return false;
-#endif
-    }
-
-    // NTFS is case insensitive, so Build and build are the same directory
-    static std::string get_effective_name(const std::string & fname) {
-#if defined(_WIN32)
-        std::string lowered = fname;
-        std::transform(lowered.begin(), lowered.end(), lowered.begin(),
-                       [](unsigned char c) { return (char) std::tolower(c); });
-        return lowered;
-#else
-        return fname;
-#endif
-    }
-
     static const std::unordered_set<std::string> & junk_dir_names() {
         static const std::unordered_set<std::string> names = {
             ".git", ".svn", ".hg", "node_modules", "__pycache__",
@@ -394,53 +337,46 @@ private:
     std::vector<list_entry> list_entries_fallback(const std::string & base, int max_depth, list_kind kind,
                                                   std::chrono::steady_clock::time_point deadline, bool & truncated) const {
         std::vector<list_entry> result;
+        std::error_code ec;
 
         std::vector<std::tuple<fs::path, fs::path, int>> stack;
-        stack.emplace_back(path_from_utf8(base), fs::path(), 0);
+        stack.emplace_back(fs::path(base), fs::path(), 0);
 
         while (!stack.empty()) {
-            if (std::chrono::steady_clock::now() >= deadline) {
-                truncated = true;
-                return result;
-            }
-
-            auto [dir, rel_dir, depth] = std::move(stack.back());
+            auto [dir, rel_dir, depth] = stack.back();
             stack.pop_back();
 
-            std::error_code ec;
-            // step the iterator by hand: the throwing increment escapes on a directory that goes away
+            // the throwing increment would escape the tool on a directory that
+            // goes away mid walk, so step the iterator explicitly
             fs::directory_iterator it(dir, fs::directory_options::skip_permission_denied, ec);
-            // permission errors are skipped above, so this is a subtree the caller never sees
-            if (ec) {
-                truncated = true;
-                continue;
-            }
             for (const fs::directory_iterator end; it != end; it.increment(ec)) {
-                if (ec) {
-                    truncated = true;
-                    break;
-                }
+                if (ec) break;
                 if (std::chrono::steady_clock::now() >= deadline) {
                     truncated = true;
                     return result;
                 }
                 const fs::directory_entry & entry = *it;
-                const fs::path fname = entry.path().filename();
+                std::string fname = entry.path().filename().string();
                 std::error_code tec;
-                const bool is_dir = entry.is_directory(tec);
-                if (tec) continue;
-                if (is_dir) {
+                if (entry.is_directory(tec)) {
+                    std::string rel = (rel_dir / fname).string();
+                    std::replace(rel.begin(), rel.end(), '\\', '/');
                     if (kind == list_kind::dirs || kind == list_kind::all) {
-                        result.push_back({path_to_utf8(rel_dir / fname), true});
+                        result.push_back({rel, true});
                     }
-                    // junk directories stay selectable but are never walked: they can be enormous
-                    if (junk_dir_names().count(get_effective_name(path_to_utf8(fname))) > 0) continue;
-                    if (!is_link(entry) && (max_depth == 0 || depth + 1 < max_depth)) {
+                    // junk directories stay selectable but are never walked: they
+                    // hold nothing worth searching and can be enormous
+                    if (junk_dir_names().count(fname) > 0) continue;
+                    // do not descend into symlinks: a link can point back to an
+                    // ancestor and loop forever
+                    if (!entry.is_symlink(tec) && (max_depth == 0 || depth + 1 < max_depth)) {
                         stack.emplace_back(entry.path(), rel_dir / fname, depth + 1);
                     }
                 } else if (entry.is_regular_file(tec)) {
+                    std::string rel = (rel_dir / fname).string();
+                    std::replace(rel.begin(), rel.end(), '\\', '/');
                     if (kind == list_kind::files || kind == list_kind::all) {
-                        result.push_back({path_to_utf8(rel_dir / fname), false});
+                        result.push_back({rel, false});
                     }
                 }
             }
@@ -458,7 +394,7 @@ static std::unique_ptr<tools_io> make_tools_io(const json & params) {
 // no '/' in pattern -> match basename at any depth; else match full relative path
 static bool path_glob_match(const std::string & pattern, const std::string & rel_path) {
     if (pattern.find('/') == std::string::npos) {
-        return glob_match(pattern, path_to_utf8(path_from_utf8(rel_path).filename()));
+        return glob_match(pattern, fs::path(rel_path).filename().string());
     }
     if (pattern == "**" || pattern.rfind("**/", 0) == 0 || pattern.rfind('/', 0) == 0) {
         return glob_match(pattern, rel_path);
@@ -555,7 +491,7 @@ struct server_tool_read_file : server_tool {
 // file_glob_search: find files matching a glob pattern under a base directory
 //
 
-static constexpr int SERVER_TOOL_FILE_SEARCH_MAX_RESULTS = 100;
+static constexpr size_t SERVER_TOOL_FILE_SEARCH_MAX_RESULTS = 100;
 static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_FILE = "file";
 static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_DIR  = "dir";
 static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_ALL  = "all";
@@ -589,7 +525,7 @@ struct server_tool_file_glob_search : server_tool {
                         {"exclude",   {{"type", "string"},  {"description", "Glob pattern for files to exclude"}}},
                         {"type",      {{"type", "string"},  {"description", "Entry type to return: \"file\" (default), \"dir\" or \"all\""}}},
                         {"max_depth", {{"type", "integer"}, {"description", "Maximum depth to descend into subdirectories (default: 0 = unlimited; 1 = direct children only)"}}},
-                        {"limit",     {{"type", "integer"}, {"description", string_format("Maximum number of results to return, capped at %d (default %d)", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS, SERVER_TOOL_FILE_SEARCH_MAX_RESULTS)}}},
+                        {"limit",     {{"type", "integer"}, {"description", string_format("Maximum number of results to return (default %zu; values below 1 fall back to the default)", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS)}}},
                     }},
                     {"required", json::array({"path"})},
                 }},
@@ -600,18 +536,17 @@ struct server_tool_file_glob_search : server_tool {
     json invoke(json params, server_tool::stream *) const override {
         auto io = make_tools_io(params);
 
-        const std::string path = params.at("path").get<std::string>();
-
-        std::string base      = io->resolve(path);
+        std::string base      = io->resolve(params.at("path").get<std::string>());
+        // normalize to forward slashes so the web UI (which assumes '/') can
+        // join the relative entries into absolute paths on Windows too
+        std::replace(base.begin(), base.end(), '\\', '/');
         std::string include   = json_value(params, "include", std::string("**"));
         std::string exclude   = json_value(params, "exclude", std::string(""));
         std::string type      = json_value(params, "type",    std::string("file"));
         int         max_depth = std::max(0, json_value(params, "max_depth", 0));
-        const int   limit_req = json_value(params, "limit", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
-        if (limit_req < 1) {
-            return {{"error", "invalid limit: " + std::to_string(limit_req) + " (expected 1 or more)"}};
-        }
-        const int   limit     = std::min(limit_req, SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
+        int         limit     = json_value(params, "limit", (int) SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
+        if (limit < 1) limit = SERVER_TOOL_FILE_SEARCH_MAX_RESULTS;
+        limit = std::min(limit, (int) SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
 
         list_kind kind;
         if (type == SERVER_TOOL_FILE_SEARCH_TYPE_FILE) {
@@ -624,13 +559,15 @@ struct server_tool_file_glob_search : server_tool {
             return {{"error", "invalid type: " + type + " (expected \"file\", \"dir\" or \"all\")"}};
         }
 
-        const auto listing = io->list_entries(base, max_depth, kind);
-        if (!listing.err.empty()) {
-            return {{"error", listing.err + ": " + path}};
+        std::string err;
+        bool truncated = false;
+        auto entries = io->list_entries(base, params.at("path").get<std::string>(), max_depth, kind, err, truncated);
+        if (!err.empty()) {
+            return {{"error", err}};
         }
 
         std::vector<tools_io::list_entry> matches;
-        for (const auto & entry : listing.entries) {
+        for (const auto & entry : entries) {
             if (!path_glob_match(include, entry.rel)) continue;
             if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue;
             matches.push_back(entry);
@@ -655,8 +592,8 @@ struct server_tool_file_glob_search : server_tool {
                 "[%zu results limit reached (%zu total matches). Refine the glob pattern to narrow the search.]\n",
                 shown, total);
         }
-        if (listing.truncated) {
-            output_text << "[results truncated: time budget or unreadable directory]\n";
+        if (truncated) {
+            output_text << "[search timed out, results truncated]\n";
         }
 
         // `base` is always absolute (resolve falls back to the server cwd), so
@@ -751,14 +688,16 @@ struct server_tool_grep_search : server_tool {
         if (io->is_regular_file(abs_path)) {
             files.emplace_back(abs_path, path);
         } else if (io->is_directory(abs_path)) {
-            const auto listing = io->list_entries(abs_path, 0, list_kind::files);
-            if (!listing.err.empty()) {
-                return {{"error", listing.err + ": " + path}};
+            std::string err;
+            bool truncated = false;
+            auto candidates = io->list_entries(abs_path, path, 0, list_kind::files, err, truncated);
+            if (!err.empty()) {
+                return {{"error", err}};
             }
-            for (const auto & entry : listing.entries) {
+            for (const auto & entry : candidates) {
                 if (!path_glob_match(include, entry.rel)) continue;
                 if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue;
-                files.emplace_back(path_to_utf8(path_from_utf8(abs_path) / path_from_utf8(entry.rel)), entry.rel);
+                files.emplace_back((fs::path(abs_path) / entry.rel).string(), entry.rel);
             }
         } else {
             return {{"error", "path does not exist: " + path}};
@@ -1367,7 +1306,7 @@ struct server_tool_get_info : server_tool {
         std::string cwd = json_value(params, "cwd", std::string());
         if (cwd.empty()) {
             std::error_code ec;
-            cwd = path_to_utf8(fs::current_path(ec));
+            cwd = fs::current_path(ec).string();
         }
 
         return {

tools/server/tests/unit/test_proxy.py

@@ -1,7 +1,5 @@
 import pytest
 from utils import *
-import threading
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 server = ServerPreset.tinyllama2()
 
@@ -41,31 +39,3 @@ def test_mcp_proxy_custom_port():
     res = server.make_request("GET", f"/cors-proxy?url=http://{server.server_host}:{server.server_port}/models")
     assert res.status_code == 200
     assert "data" in res.body
-
-
-def test_mcp_proxy_no_content():
-    # note: see issue #26598
-    class NoContentHandler(BaseHTTPRequestHandler):
-        def do_POST(self):
-            self.send_response(204)
-            self.end_headers()
-
-        def log_message(self, format, *args):
-            pass
-
-    target = ThreadingHTTPServer(("127.0.0.1", 0), NoContentHandler)
-    target_thread = threading.Thread(target=target.serve_forever, daemon=True)
-    target_thread.start()
-
-    try:
-        global server
-        server.ui_mcp_proxy = True
-        server.start()
-
-        res = server.make_request("POST", f"/cors-proxy?url=http://127.0.0.1:{target.server_port}/", data={})
-        assert res.status_code == 204
-        assert res.body in (None, b"", "")
-    finally:
-        target.shutdown()
-        target.server_close()
-

tools/server/tests/unit/test_router.py

@@ -145,156 +145,6 @@ def test_router_models_max_evicts_lru():
     assert _get_model_status(first) == "unloaded"
 
 
-# server_lru_sched tests (relying on LLAMA_SERVER_DEBUG_FAKE_TIMING)
-
-MODEL_A = "ggml-org/tinygemma3-GGUF:Q8_0"
-MODEL_B = "ggml-org/test-model-stories260K:F32"
-MODEL_C = "ggml-org/test-model-stories260K-infill:F32"
-
-
-def _tokenize(model_id: str, timeout: float | None = DEFAULT_REQUEST_TIMEOUT) -> ServerResponse:
-    return server.make_request(
-        "POST", "/tokenize", data={"model": model_id, "content": "hello world"}, timeout=timeout
-    )
-
-
-class _Bg:
-    """runs one request in a thread, keeps its result, error and finish time"""
-
-    def __init__(self, fn):
-        self.result = None
-        self.error: Exception | None = None
-        self.done_at: float = 0.0
-        self._thread = threading.Thread(target=self._run, args=(fn,), daemon=True)
-
-    def _run(self, fn):
-        try:
-            self.result = fn()
-        except Exception as e:
-            self.error = e
-        self.done_at = time.time()
-
-    def start(self):
-        self._thread.start()
-        return self
-
-    def join(self, timeout: int = 180):
-        self._thread.join(timeout)
-        assert not self._thread.is_alive(), "background request did not finish in time"
-        return self
-
-    def assert_ok(self, what: str):
-        assert self.error is None, f"{what} raised {self.error!r}"
-        assert self.result is not None and self.result.status_code == 200, \
-            f"{what} failed: {self.result.status_code if self.result else None} {self.result.body if self.result else None}"
-
-
-def test_router_queue_does_not_evict_busy_model():
-    """a request that finds no free slot waits, and the model serving a request survives it"""
-    global server
-    server.models_max = 1
-    server.start()
-
-    _load_model_and_wait(MODEL_A, timeout=120)
-
-    busy = _Bg(lambda: _tokenize(MODEL_A)).start()
-    time.sleep(0.5)  # let the request reach the child and take the only slot
-
-    # no slot free and MODEL_A is busy, so this queues instead of evicting mid-request
-    queued = _Bg(lambda: _tokenize(MODEL_B)).start()
-
-    busy.join()
-    queued.join()
-
-    # had MODEL_A been evicted while serving, its own request would have died
-    busy.assert_ok("request against the busy model")
-    queued.assert_ok("queued request")
-
-    _wait_for_model_status(MODEL_B, {"loaded"}, timeout=120)
-    assert _get_model_status(MODEL_A) == "unloaded"
-
-
-def test_router_queue_coalesces_requests_for_same_model():
-    """many requests for one missing model share a slot, so only one model is given up"""
-    global server
-    server.models_max = 2
-    server.start()
-
-    _load_model_and_wait(MODEL_A, timeout=120)
-    _load_model_and_wait(MODEL_B, timeout=120)
-
-    # keep MODEL_A busy so MODEL_B is the only model that can be given up
-    busy = _Bg(lambda: _tokenize(MODEL_A)).start()
-    time.sleep(0.5)
-
-    waiters = [_Bg(lambda: _tokenize(MODEL_C)).start() for _ in range(3)]
-
-    busy.join()
-    for w in waiters:
-        w.join()
-
-    busy.assert_ok("request against the busy model")
-    for i, w in enumerate(waiters):
-        w.assert_ok(f"queued request {i}")
-
-    _wait_for_model_status(MODEL_C, {"loaded"}, timeout=120)
-    # one entry for 3 requests means one eviction: MODEL_B goes, MODEL_A is left alone.
-    # without coalescing the leftover entries still ask for a slot,
-    # and MODEL_A is taken too as soon as it goes idle
-    assert _get_model_status(MODEL_A) == "loaded"
-    assert _get_model_status(MODEL_B) == "unloaded"
-
-
-def test_router_queue_client_disconnect_keeps_model():
-    """a client that leaves while queued must not cost a running model its slot"""
-    global server
-    server.models_max = 1
-    server.start()
-
-    _load_model_and_wait(MODEL_A, timeout=120)
-
-    busy = _Bg(lambda: _tokenize(MODEL_A)).start()
-    time.sleep(0.5)
-
-    # queues behind MODEL_A, then gives up long before MODEL_A goes idle
-    with pytest.raises(requests.exceptions.RequestException):
-        _tokenize(MODEL_B, timeout=1)
-
-    busy.join()
-    busy.assert_ok("request against the busy model")
-
-    # nobody is waiting anymore, so MODEL_A keeps its slot
-    time.sleep(3)
-    assert _get_model_status(MODEL_A) == "loaded"
-    assert _get_model_status(MODEL_B) == "unloaded"
-
-
-def test_router_queue_is_fifo():
-    """the queue is served in arrival order"""
-    global server
-    server.models_max = 1
-    server.start()
-
-    _load_model_and_wait(MODEL_A, timeout=120)
-
-    busy = _Bg(lambda: _tokenize(MODEL_A)).start()
-    time.sleep(0.5)
-
-    first = _Bg(lambda: _tokenize(MODEL_B)).start()
-    time.sleep(1)  # keep the arrival order unambiguous
-    second = _Bg(lambda: _tokenize(MODEL_C)).start()
-
-    busy.join()
-    first.join()
-    second.join()
-
-    busy.assert_ok("request against the busy model")
-    first.assert_ok("first queued request")
-    second.assert_ok("second queued request")
-
-    assert first.done_at < second.done_at, "queue was not served in arrival order"
-
-
 def test_router_no_models_autoload():
     global server
     server.no_models_autoload = True

tools/server/tests/unit/test_tools_builtin.py

@@ -214,27 +214,6 @@ def test_tools_builtin_file_glob_search_max_depth_and_limit(tmp_path):
     assert "Total matches: 3" in res["plain_text_response"]
 
 
-def test_tools_builtin_file_glob_search_junk_dirs(tmp_path):
-    global server
-    server.start()
-
-    (tmp_path / "build" / "nested").mkdir(parents=True)
-    (tmp_path / "build" / "artifact.txt").write_text("built")
-    (tmp_path / "src").mkdir()
-    (tmp_path / "src" / "main.cpp").write_text("int main() {}")
-
-    # a junk directory stays selectable as a working directory
-    res = call_tool("file_glob_search", {"path": str(tmp_path), "type": "dir", "max_depth": 1})
-    assert "build" in [e["path"] for e in res["entries"]]
-
-    # but it is never walked, so nothing inside it shows up
-    res = call_tool("file_glob_search", {"path": str(tmp_path), "type": "all"})
-    paths = [e["path"] for e in res["entries"]]
-    assert "src/main.cpp" in paths
-    assert "build/artifact.txt" not in paths
-    assert "build/nested" not in paths
-
-
 def test_tools_builtin_file_glob_search_rejects_invalid_type(tmp_path):
     global server
     server.start()

tools/server/tests/utils.py

@@ -132,10 +132,7 @@ class ServerProcess:
         self.external_server = "DEBUG_EXTERNAL" in os.environ
 
     def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None:
-        env = {
-            **os.environ,
-            "LLAMA_SERVER_DEBUG_FAKE_TIMING": "1",
-        }
+        env = {**os.environ}
         if "LLAMA_CACHE" not in os.environ:
             env["LLAMA_CACHE"] = "tmp"
         if self.external_server:

tools/ui/.npmrc

@@ -1,3 +1,2 @@
 engine-strict=true
 ignore-scripts=true
-min-release-age=7

tools/ui/.storybook/main.ts

@@ -11,8 +11,7 @@ const config: StorybookConfig = {
 		'@chromatic-com/storybook',
 		'@storybook/addon-vitest',
 		'@storybook/addon-a11y',
-		'@storybook/addon-docs',
-		'@storybook/addon-mcp'
+		'@storybook/addon-docs'
 	],
 	framework: '@storybook/sveltekit',
 	viteFinal: async (config) => {

tools/ui/.storybook/vitest.setup.ts

@@ -0,0 +1,12 @@
+import * as a11yAddonAnnotations from '@storybook/addon-a11y/preview';
+import { setProjectAnnotations } from '@storybook/sveltekit';
+import * as previewAnnotations from './preview';
+import { beforeAll } from 'vitest';
+
+const project = setProjectAnnotations([a11yAddonAnnotations, previewAnnotations]);
+
+beforeAll(async () => {
+	if (project.beforeAll) {
+		await project.beforeAll();
+	}
+});

tools/ui/package.json

@@ -27,20 +27,20 @@
 		"cleanup": "rm -rf .svelte-kit build node_modules test-results dist dev-dist debug-storybook.log static/pwa-*.png static/maskable-icon-*.png static/apple-touch-icon-*.png static/apple-splash-*.png static/favicon*.ico"
 	},
 	"devDependencies": {
-		"@chromatic-com/storybook": "5.2.1",
+		"@chromatic-com/storybook": "5.0.0",
 		"@eslint/compat": "1.4.1",
 		"@eslint/js": "9.39.2",
 		"@internationalized/date": "3.12.2",
 		"@lucide/svelte": "1.25.0",
-		"@modelcontextprotocol/sdk": "1.30.0",
+		"@modelcontextprotocol/sdk": "1.26.0",
 		"@playwright/test": "1.56.1",
-		"@storybook/addon-a11y": "10.5.6",
-		"@storybook/addon-docs": "10.5.6",
-		"@storybook/addon-svelte-csf": "5.1.2",
-		"@storybook/addon-vitest": "10.5.6",
-		"@storybook/sveltekit": "10.5.6",
+		"@storybook/addon-a11y": "10.2.4",
+		"@storybook/addon-docs": "10.2.4",
+		"@storybook/addon-svelte-csf": "5.0.10",
+		"@storybook/addon-vitest": "10.2.4",
+		"@storybook/sveltekit": "10.2.4",
 		"@sveltejs/adapter-static": "3.0.10",
-		"@sveltejs/kit": "2.70.2",
+		"@sveltejs/kit": "2.60.1",
 		"@sveltejs/vite-plugin-svelte": "6.2.1",
 		"@tailwindcss/forms": "0.5.10",
 		"@tailwindcss/typography": "0.5.16",
@@ -48,16 +48,16 @@
 		"@types/node": "24.13.0",
 		"@vite-pwa/assets-generator": "1.0.2",
 		"@vite-pwa/sveltekit": "1.1.0",
-		"@vitest/browser": "4.1.10",
-		"@vitest/browser-playwright": "4.1.10",
-		"@vitest/coverage-v8": "4.1.10",
+		"@vitest/browser": "4.1.8",
+		"@vitest/browser-playwright": "4.1.8",
+		"@vitest/coverage-v8": "4.1.8",
 		"bits-ui": "2.18.1",
 		"clsx": "2.1.1",
 		"dexie": "4.4.3",
-		"dompurify": "3.4.13",
+		"dompurify": "3.4.11",
 		"eslint": "9.39.4",
 		"eslint-config-prettier": "10.1.8",
-		"eslint-plugin-storybook": "10.5.6",
+		"eslint-plugin-storybook": "10.4.2",
 		"eslint-plugin-svelte": "3.19.0",
 		"fflate": "0.8.3",
 		"globals": "16.5.0",
@@ -82,7 +82,7 @@
 		"remark-math": "6.0.0",
 		"remark-rehype": "11.1.2",
 		"sass": "1.100.0",
-		"storybook": "10.5.6",
+		"storybook": "10.4.2",
 		"svelte": "5.56.1",
 		"svelte-check": "4.6.0",
 		"svelte-sonner": "1.1.1",
@@ -95,16 +95,13 @@
 		"unified": "11.0.5",
 		"unist-util-visit": "5.1.0",
 		"uuid": "13.0.2",
-		"vite": "7.3.6",
+		"vite": "7.3.5",
 		"vite-plugin-devtools-json": "0.2.1",
-		"vitest": "4.1.10",
+		"vitest": "4.1.8",
 		"vitest-browser-svelte": "2.1.1",
-		"workbox-window": "7.4.1",
-		"@storybook/addon-mcp": "0.7.0"
+		"workbox-window": "7.4.1"
 	},
 	"overrides": {
-		"cookie": "1.1.1",
-		"sharp": "0.35.3",
-		"valibot": "1.4.2"
+		"cookie": "1.1.1"
 	}
 }

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectory.svelte

@@ -87,9 +87,8 @@
 	let searchSeq = 0;
 
 	// Cache of the last file_glob_search result per (parent, include, max_depth),
-	// so repeated queries in the same directory don't re-walk the tree. Entering
-	// a directory hits it every time: the children listed for an exactly typed
-	// segment are what the next keystroke, the trailing slash, asks for again.
+	// so repeated queries in the same directory don't re-walk the tree. Entries
+	// expire after a short TTL.
 	const SEARCH_CACHE_TTL_MS = 2000;
 	const searchCache = new SvelteMap<string, { results: GlobEntry[]; base: string; at: number }>();
 
@@ -162,11 +161,7 @@
 		if (typeof res.error === 'string') return { base: '', entries: [], error: res.error };
 		const base = typeof res.base === 'string' ? res.base : '';
 		const entries = Array.isArray(res.entries) ? (res.entries as GlobEntry[]) : [];
-		const now = Date.now();
-		for (const [k, v] of searchCache) {
-			if (now - v.at >= SEARCH_CACHE_TTL_MS) searchCache.delete(k);
-		}
-		searchCache.set(key, { results: entries, base, at: now });
+		searchCache.set(key, { results: entries, base, at: Date.now() });
 		return { base, entries };
 	}
 

tools/ui/src/lib/components/app/dialogs/DialogModelNotAvailable.svelte

@@ -1,6 +1,5 @@
 <script lang="ts">
 	import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
-	import { URL_PARAMS } from '$lib/constants';
 	import * as AlertDialog from '$lib/components/ui/alert-dialog';
 	import { AlertTriangle, ArrowRight } from '@lucide/svelte';
 	import { goto } from '$app/navigation';
@@ -23,7 +22,7 @@
 	function handleSelectModel(model: string) {
 		// Build URL with selected model, preserving other params
 		const url = new URL(page.url);
-		url.searchParams.set(URL_PARAMS.MODEL, model);
+		url.searchParams.set('model', model);
 
 		handleOpenChange(false);
 		goto(url.toString());

tools/ui/src/lib/constants/routes.ts

@@ -1,14 +1,4 @@
-/** Query params the chat routes read from the URL. */
-export const URL_PARAMS = {
-	/** Prompt to send on arrival. */
-	QUERY: 'q',
-	/** Model to select. */
-	MODEL: 'model',
-	/** Load the selected model instead of waiting for the first message. */
-	LOAD: 'load',
-	/** Start a new chat. */
-	NEW_CHAT: 'new_chat'
-} as const;
+export const NEW_CHAT_PARAM = 'new_chat';
 
 /** Settings section slugs — used for routes and navigation. */
 export const SETTINGS_SECTION_SLUGS = {
@@ -26,7 +16,7 @@ export const ROUTES = {
 	/** Root — start of the app. */
 	START: '#/',
 	/** New chat — root with new chat query param. */
-	NEW_CHAT: `?${URL_PARAMS.NEW_CHAT}=true#/`,
+	NEW_CHAT: `?${NEW_CHAT_PARAM}=true#/`,
 	/** Chat base — for dynamic chat URLs use RouterService. */
 	CHAT: '#/chat',
 	/** MCP servers. */

tools/ui/src/lib/constants/settings-registry.ts

@@ -198,7 +198,7 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
 				key: SETTINGS_KEYS.SHOW_MESSAGE_STATS,
 				label: 'Show message generation statistics',
 				help: 'Display generation statistics (tokens/second, token count, duration) below each assistant message.',
-				defaultValue: true,
+				defaultValue: false,
 				type: SettingsFieldType.CHECKBOX,
 				section: SETTINGS_SECTION_SLUGS.DISPLAY
 			},

tools/ui/src/lib/stores/models.svelte.ts

@@ -1,12 +1,7 @@
 import { base } from '$app/paths';
 import { SvelteMap, SvelteSet } from 'svelte/reactivity';
 import { toast } from 'svelte-sonner';
-import {
-	ServerModelStatus,
-	ServerModelsSseEventType,
-	ModelModality,
-	FileTypeCategory
-} from '$lib/enums';
+import { ServerModelStatus, ServerModelsSseEventType, ModelModality } from '$lib/enums';
 import { ModelsService } from '$lib/services/models.service';
 import { PropsService } from '$lib/services/props.service';
 import { serverStore, isRouterMode } from '$lib/stores/server.svelte';
@@ -399,7 +394,6 @@ class ModelsStore {
 				model: modelId,
 				description: details?.description,
 				capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value)),
-				modalities: this.buildArchitectureModalities(item.architecture),
 				details: details?.details,
 				meta: item.meta ?? null,
 				parsedId: ModelsService.parseModelId(modelId),
@@ -1006,21 +1000,6 @@ class ModelsStore {
 		};
 	}
 
-	/** Map the router modalities, the only source available while a model is not loaded. */
-	private buildArchitectureModalities(
-		architecture: ApiModelDataEntry['architecture']
-	): ModelModalities | undefined {
-		if (!architecture) return undefined;
-
-		const inputs = architecture.input_modalities;
-
-		return {
-			vision: inputs.includes(FileTypeCategory.IMAGE),
-			audio: inputs.includes(FileTypeCategory.AUDIO),
-			video: inputs.includes(FileTypeCategory.VIDEO)
-		};
-	}
-
 	clear(): void {
 		this.unsubscribeStatus();
 		this.statusWaiters.forEach((waiter) => waiter.reject(new Error('Models store cleared')));

tools/ui/src/lib/types/api.d.ts

@@ -98,21 +98,10 @@ export interface ApiModelDataEntry {
 	aliases?: string[];
 	/** Informational tags for this model */
 	tags?: string[];
-	/** Modality capabilities, reported by the router for every model regardless of load state */
-	architecture?: ApiModelArchitecture;
 	/** Legacy meta field (may be present in older responses) */
 	meta?: Record<string, unknown> | null;
 }
 
-/**
- * Modality capabilities of a model, as advertised by the ROUTER /models endpoint.
- * Read from the model manifest, so it is available before the model is loaded.
- */
-export interface ApiModelArchitecture {
-	/** Accepted input modalities, always contains "text" */
-	input_modalities: string[];
-}
-
 /**
  * Load stage reported by the /models/sse feed, in load order.
  */

tools/ui/src/routes/(chat)/+page.svelte

@@ -3,16 +3,14 @@
 	import { chatStore } from '$lib/stores/chat.svelte';
 	import { conversationsStore, isConversationsInitialized } from '$lib/stores/conversations.svelte';
 	import { modelsStore, modelOptions } from '$lib/stores/models.svelte';
-	import { isRouterMode } from '$lib/stores/server.svelte';
 	import { onMount } from 'svelte';
 	import { page } from '$app/state';
 	import { replaceState } from '$app/navigation';
-	import { APP_NAME, URL_PARAMS } from '$lib/constants';
+	import { APP_NAME, NEW_CHAT_PARAM } from '$lib/constants';
 
-	let qParam = $derived(page.url.searchParams.get(URL_PARAMS.QUERY));
-	let modelParam = $derived(page.url.searchParams.get(URL_PARAMS.MODEL));
-	let newChatParam = $derived(page.url.searchParams.get(URL_PARAMS.NEW_CHAT));
-	let loadParam = $derived(page.url.searchParams.get(URL_PARAMS.LOAD));
+	let qParam = $derived(page.url.searchParams.get('q'));
+	let modelParam = $derived(page.url.searchParams.get('model'));
+	let newChatParam = $derived(page.url.searchParams.get(NEW_CHAT_PARAM));
 
 	// Dialog state for model not available error
 	let showModelNotAvailable = $state(false);
@@ -25,10 +23,9 @@
 	function clearUrlParams() {
 		const url = new URL(page.url);
 
-		url.searchParams.delete(URL_PARAMS.QUERY);
-		url.searchParams.delete(URL_PARAMS.MODEL);
-		url.searchParams.delete(URL_PARAMS.LOAD);
-		url.searchParams.delete(URL_PARAMS.NEW_CHAT);
+		url.searchParams.delete('q');
+		url.searchParams.delete('model');
+		url.searchParams.delete(NEW_CHAT_PARAM);
 
 		replaceState(url.toString(), {});
 	}
@@ -42,14 +39,6 @@
 			if (model) {
 				try {
 					await modelsStore.selectModelById(model.id);
-
-					// with ?load=true, start loading right away so the model is ready sooner;
-					// not awaited, so the UI stays usable during the load
-					if (loadParam === 'true' && isRouterMode() && !modelsStore.isModelLoaded(model.id)) {
-						modelsStore
-							.loadModel(model.id)
-							.catch((error) => console.error('Failed to load model:', error));
-					}
 				} catch (error) {
 					console.error('Failed to select model:', error);
 					requestedModelName = modelParam;

tools/ui/src/routes/(chat)/chat/[id]/+page.svelte

@@ -3,7 +3,7 @@
 	import { page } from '$app/state';
 	import { afterNavigate } from '$app/navigation';
 	import { DialogModelNotAvailable } from '$lib/components/app';
-	import { APP_NAME, ROUTES, URL_PARAMS } from '$lib/constants';
+	import { APP_NAME, ROUTES } from '$lib/constants';
 	import { chatStore } from '$lib/stores/chat.svelte';
 	import { conversationsStore, activeConversation } from '$lib/stores/conversations.svelte';
 	import { modelsStore, modelOptions } from '$lib/stores/models.svelte';
@@ -12,8 +12,8 @@
 	let currentChatId: string | undefined = undefined;
 
 	// URL parameters for prompt and model selection
-	let qParam = $derived(page.url.searchParams.get(URL_PARAMS.QUERY));
-	let modelParam = $derived(page.url.searchParams.get(URL_PARAMS.MODEL));
+	let qParam = $derived(page.url.searchParams.get('q'));
+	let modelParam = $derived(page.url.searchParams.get('model'));
 
 	// Dialog state for model not available error
 	let showModelNotAvailable = $state(false);
@@ -28,8 +28,8 @@
 	 */
 	function clearUrlParams() {
 		const url = new URL(page.url);
-		url.searchParams.delete(URL_PARAMS.QUERY);
-		url.searchParams.delete(URL_PARAMS.MODEL);
+		url.searchParams.delete('q');
+		url.searchParams.delete('model');
 		replaceState(url.toString(), {});
 	}
 

tools/ui/vite.config.ts

@@ -76,7 +76,8 @@ export default defineConfig({
 				extends: './vite.config.ts',
 				test: {
 					name: 'ui',
-					browser: { ...browserBaseConfig, instances: [{ browser: 'chromium', headless: true }] }
+					browser: { ...browserBaseConfig, instances: [{ browser: 'chromium', headless: true }] },
+					setupFiles: ['./.storybook/vitest.setup.ts']
 				},
 				plugins: [
 					storybookTest({

vendor/sheredom/patch-bsd.patch

@@ -1,19 +0,0 @@
-Fix build on NetBSD, which provides posix_spawn_file_actions_addchdir()
-but not the _np() variant.
-
-Upstream PR: https://github.com/sheredom/subprocess.h/pull/102
-Applied locally by scripts/sync_vendor.py until it is merged upstream.
-
-diff --git a/subprocess.h b/subprocess.h
-index 5e809023a4..74a4e006c7 100644
---- a/subprocess.h
-+++ b/subprocess.h
-@@ -1205,7 +1205,7 @@ cleanup:
- 
-   // Set working directory
-   if (process_cwd) {
--#if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000
-+#if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)
-     posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);
- #else
- #if defined(__APPLE__) && defined(__clang__)

vendor/sheredom/patch-glibc-older-than-2.29.patch

@@ -1,47 +0,0 @@
-Fix building against glibc older than 2.29, which has no
-posix_spawn_file_actions_addchdir_np (the symbol is genuinely absent from
-libc.so, so no feature-test macro helps). Affects manylinux2014 (glibc 2.17)
-and manylinux_2_28, and was reported on RHEL 8.1. A requested process_cwd now
-fails with ENOSYS there instead of failing the build.
-
-Upstream PR: https://github.com/sheredom/subprocess.h/pull/104
-Applied locally by scripts/sync_vendor.py until it is merged upstream.
-
-(the README.md and test/ changes from the PR are omitted, we only vendor
-subprocess.h; rebased on top of patch-bsd.patch, so apply it after that one)
-
-diff --git a/subprocess.h b/subprocess.h
-index 1ef424a..c363393 100644
---- a/subprocess.h
-+++ b/subprocess.h
-@@ -274,6 +274,21 @@ subprocess_weak int subprocess_alive(struct subprocess_s *const process);
- #include <unistd.h>
- #endif
- 
-+/* Whether subprocess_create_ex can honour process_cwd. glibc only gained
-+   posix_spawn_file_actions_addchdir_np in 2.29. Define this yourself to
-+   override the detection, for instance on musl older than 1.1.24. */
-+#if !defined(SUBPROCESS_HAVE_CWD)
-+#if defined(__GLIBC__)
-+#if __GLIBC_PREREQ(2, 29)
-+#define SUBPROCESS_HAVE_CWD 1
-+#else
-+#define SUBPROCESS_HAVE_CWD 0
-+#endif
-+#else
-+#define SUBPROCESS_HAVE_CWD 1
-+#endif
-+#endif
-+
- #if defined(_WIN32)
- 
- #include <wchar.h>
-@@ -1219,6 +1234,8 @@ cleanup:
-   if (process_cwd) {
- #if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)
-     posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);
-+#elif !SUBPROCESS_HAVE_CWD
-+    posix_error = ENOSYS;
- #else
- #if defined(__APPLE__) && defined(__clang__)
- #pragma clang diagnostic push

vendor/sheredom/patch-windows-quote-backslash.patch

@@ -1,107 +0,0 @@
-Fix Windows command line quoting of backslash runs: a trailing backslash, or
-backslashes preceding a double quote, were not doubled, so CommandLineToArgvW
-in the child parsed them as escapes and mangled the argument list.
-
-Upstream PR: https://github.com/sheredom/subprocess.h/pull/101
-Applied locally by scripts/sync_vendor.py until it is merged upstream.
-
-(the test/ changes from the PR are omitted, we only vendor subprocess.h)
-
-diff --git a/subprocess.h b/subprocess.h
-index 5e80902..b06ad4d 100644
---- a/subprocess.h
-+++ b/subprocess.h
-@@ -653,6 +653,7 @@ int subprocess_create_ex(const char *const commandLine[], int options,
-   int wide_len;
-   int i, j;
-   int need_quoting;
-+  subprocess_size_t bs_run;
-   unsigned long flags = 0;
-   unsigned long last_error = 0;
-   int result = subprocess_error_unknown;
-@@ -906,25 +907,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,
-     len++;
- 
-     // Quote the argument if it has a space in it
--    if (strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||
--        commandLine[i][0] == SUBPROCESS_NULL)
-+    need_quoting = strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||
-+                   commandLine[i][0] == SUBPROCESS_NULL;
-+    if (need_quoting)
-       len += 2;
- 
-+    bs_run = 0;
-     for (j = 0; '\0' != commandLine[i][j]; j++) {
--      switch (commandLine[i][j]) {
--      default:
--        break;
--      case '\\':
--        if (commandLine[i][j + 1] == '"') {
--          len++;
--        }
-+      len++;
- 
--        break;
--      case '"':
--        len++;
--        break;
-+      if ('\\' == commandLine[i][j]) {
-+        bs_run++;
-+      } else {
-+        if ('"' == commandLine[i][j]) {
-+          // Duplicate the preceding run and escape the quote.
-+          len += bs_run + 1;
-+        }
-+        bs_run = 0;
-       }
--      len++;
-+    }
-+
-+    if (need_quoting) {
-+      // Duplicate trailing slashes before the generated closing quote.
-+      len += bs_run;
-     }
-   }
- 
-@@ -949,22 +954,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,
-       commandLineCombined[len++] = '"';
-     }
- 
--    for (j = 0; '\0' != commandLine[i][j]; j++) {
--      switch (commandLine[i][j]) {
--      default:
--        break;
--      case '\\':
--        if (commandLine[i][j + 1] == '"') {
--          commandLineCombined[len++] = '\\';
--        }
-+    for (j = 0; '\0' != commandLine[i][j];) {
-+      bs_run = 0;
-+      while ('\\' == commandLine[i][j]) {
-+        bs_run++;
-+        j++;
-+      }
-+
-+      if ('"' == commandLine[i][j]) {
-+        // 2n + 1 slashes preserve n slashes and escape the quote.
-+        bs_run = (bs_run * 2) + 1;
-+      } else if ('\0' == commandLine[i][j] && need_quoting) {
-+        // 2n slashes preserve n slashes before the closing quote.
-+        bs_run *= 2;
-+      }
- 
--        break;
--      case '"':
-+      while (bs_run > 0) {
-         commandLineCombined[len++] = '\\';
--        break;
-+        bs_run--;
-       }
- 
--      commandLineCombined[len++] = commandLine[i][j];
-+      if ('\0' != commandLine[i][j]) {
-+        commandLineCombined[len++] = commandLine[i][j++];
-+      }
-     }
-     if (need_quoting) {
-       commandLineCombined[len++] = '"';

vendor/sheredom/subprocess.h

@@ -274,21 +274,6 @@ subprocess_weak int subprocess_alive(struct subprocess_s *const process);
 #include <unistd.h>
 #endif
 
-/* Whether subprocess_create_ex can honour process_cwd. glibc only gained
-   posix_spawn_file_actions_addchdir_np in 2.29. Define this yourself to
-   override the detection, for instance on musl older than 1.1.24. */
-#if !defined(SUBPROCESS_HAVE_CWD)
-#if defined(__GLIBC__)
-#if __GLIBC_PREREQ(2, 29)
-#define SUBPROCESS_HAVE_CWD 1
-#else
-#define SUBPROCESS_HAVE_CWD 0
-#endif
-#else
-#define SUBPROCESS_HAVE_CWD 1
-#endif
-#endif
-
 #if defined(_WIN32)
 
 #include <wchar.h>
@@ -668,7 +653,6 @@ int subprocess_create_ex(const char *const commandLine[], int options,
   int wide_len;
   int i, j;
   int need_quoting;
-  subprocess_size_t bs_run;
   unsigned long flags = 0;
   unsigned long last_error = 0;
   int result = subprocess_error_unknown;
@@ -922,29 +906,25 @@ int subprocess_create_ex(const char *const commandLine[], int options,
     len++;
 
     // Quote the argument if it has a space in it
-    need_quoting = strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||
-                   commandLine[i][0] == SUBPROCESS_NULL;
-    if (need_quoting)
+    if (strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||
+        commandLine[i][0] == SUBPROCESS_NULL)
       len += 2;
 
-    bs_run = 0;
     for (j = 0; '\0' != commandLine[i][j]; j++) {
-      len++;
-
-      if ('\\' == commandLine[i][j]) {
-        bs_run++;
-      } else {
-        if ('"' == commandLine[i][j]) {
-          // Duplicate the preceding run and escape the quote.
-          len += bs_run + 1;
+      switch (commandLine[i][j]) {
+      default:
+        break;
+      case '\\':
+        if (commandLine[i][j + 1] == '"') {
+          len++;
         }
-        bs_run = 0;
-      }
-    }
 
-    if (need_quoting) {
-      // Duplicate trailing slashes before the generated closing quote.
-      len += bs_run;
+        break;
+      case '"':
+        len++;
+        break;
+      }
+      len++;
     }
   }
 
@@ -969,29 +949,22 @@ int subprocess_create_ex(const char *const commandLine[], int options,
       commandLineCombined[len++] = '"';
     }
 
-    for (j = 0; '\0' != commandLine[i][j];) {
-      bs_run = 0;
-      while ('\\' == commandLine[i][j]) {
-        bs_run++;
-        j++;
-      }
+    for (j = 0; '\0' != commandLine[i][j]; j++) {
+      switch (commandLine[i][j]) {
+      default:
+        break;
+      case '\\':
+        if (commandLine[i][j + 1] == '"') {
+          commandLineCombined[len++] = '\\';
+        }
 
-      if ('"' == commandLine[i][j]) {
-        // 2n + 1 slashes preserve n slashes and escape the quote.
-        bs_run = (bs_run * 2) + 1;
-      } else if ('\0' == commandLine[i][j] && need_quoting) {
-        // 2n slashes preserve n slashes before the closing quote.
-        bs_run *= 2;
-      }
-
-      while (bs_run > 0) {
+        break;
+      case '"':
         commandLineCombined[len++] = '\\';
-        bs_run--;
+        break;
       }
 
-      if ('\0' != commandLine[i][j]) {
-        commandLineCombined[len++] = commandLine[i][j++];
-      }
+      commandLineCombined[len++] = commandLine[i][j];
     }
     if (need_quoting) {
       commandLineCombined[len++] = '"';
@@ -1232,10 +1205,8 @@ cleanup:
 
   // Set working directory
   if (process_cwd) {
-#if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)
+#if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000
     posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);
-#elif !SUBPROCESS_HAVE_CWD
-    posix_error = ENOSYS;
 #else
 #if defined(__APPLE__) && defined(__clang__)
 #pragma clang diagnostic push