Revert "CUDA: size routed MoE MMQ N-tiles from typical expert width on RDNA3 (#24546 )" (#28551 )

This reverts commit 0c963452ea. Assisted-by: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Q7rfnkjzgnfhvJsdeXhdoH
nix : update deprecated expressions (#28145 )
2026-09-07 16:37:57 +02:00 · 2026-09-07 16:28:19 +02:00 · 2026-09-07 15:59:45 +02:00 · 2026-09-07 15:55:14 +02:00 · 2026-09-07 15:50:46 +02:00 · 2026-09-07 15:24:14 +02:00
252 changed files with 35073 additions and 1947 deletions
@@ -31,7 +31,7 @@
    ]
    && blas.meta.available,
  useCuda ? config.cudaSupport,
-  useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin,
+  useMetalKit ? stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isDarwin,
  # Increases the runtime closure size by ~700M
  useMpi ? false,
  useRocm ? config.rocmSupport,
@@ -92,7 +92,7 @@ let

  cudaBuildInputs = with cudaPackages; [
    cuda_cudart
-    cuda_cccl # <nv/target>
+    cccl # <nv/target>
    libcublas
  ];

@@ -166,7 +166,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
  # `xcrun` is used find the path of the Metal compiler, which is varible
  # and not on $PATH
  # see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
-  __noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
+  __noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;

  nativeBuildInputs =
    [
@@ -181,10 +181,10 @@ effectiveStdenv.mkDerivation (finalAttrs: {
      autoAddDriverRunpath
    ]
    ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
-    ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
+    ++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];

  buildInputs =
-    optionals effectiveStdenv.isDarwin darwinBuildInputs
+    optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
    ++ optionals useCuda cudaBuildInputs
    ++ optionals useMpi [ mpi ]
    ++ optionals useRocm rocmBuildInputs
@@ -245,7 +245,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {

    # Configurations that are known to result in build failures. Can be
    # overridden by importing Nixpkgs with `allowBroken = true`.
-    broken = (useMetalKit && !effectiveStdenv.isDarwin);
+    broken = (useMetalKit && !effectiveStdenv.hostPlatform.isDarwin);

    description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
    homepage = "https://github.com/ggml-org/llama.cpp/";
@@ -1,4 +1,4 @@
-blank_issues_enabled: true
+blank_issues_enabled: false
 contact_links:
  - name: Got an idea?
    url: https://github.com/ggml-org/llama.cpp/discussions/categories/ideas
@@ -50,8 +50,16 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: apple-arm64
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: apple-arm64
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache

      - name: Build
        id: cmake_build
@@ -67,6 +75,18 @@ jobs:
            -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
          time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)

+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: apple-arm64
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
      - name: Check for leaks
        run: |
          cmd=(./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1)
@@ -80,16 +100,6 @@ jobs:
          cd build
          ctest -L main -E "test-llama-archs" --verbose --timeout 900

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: apple-arm64
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
-
  macos-latest-x64:
    runs-on: macos-15-intel

@@ -102,8 +112,16 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: apple-x64
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: apple-x64
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache

      - name: Build
        id: cmake_build
@@ -120,22 +138,24 @@ jobs:
            -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
          time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)

+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: apple-x64
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
      - name: Test
        id: cmake_test
        run: |
          cd build
          ctest -L main --verbose --timeout 900

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: apple-x64
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
-
  macos-latest-ios-xcode:
    runs-on: macos-latest

@@ -65,8 +65,7 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: cpu-${{ matrix.os }}
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false

      - name: Build Dependencies
        id: build_depends
@@ -91,6 +90,15 @@ jobs:
          python3 -m pip install --upgrade pip setuptools
          pip3 install ./gguf-py

+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: cpu-${{ matrix.os }}
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache
+
      - name: Build
        id: cmake_build
        run: |
@@ -100,6 +108,18 @@ jobs:
            -DGGML_RPC=ON
          time cmake --build build --config Release -j $(nproc)

+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: cpu-${{ matrix.os }}
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
      - name: Test
        id: cmake_test
        run: |
@@ -117,18 +137,6 @@ jobs:
          ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
          ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256

-      # note: real deletion only on push to master (same condition as the ccache save),
-      #       dry-run otherwise (the token is read-only on PRs from forks)
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: cpu-${{ matrix.os }}
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
-
  windows:
    name: windows / ${{ matrix.build }}
    runs-on: windows-2025
@@ -61,7 +61,7 @@ jobs:
      - name: ccache-buckets-restore
        uses: ./.github/actions/ccache-buckets
        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
        with:
          key: cuda-ubuntu-24.04-cuda
          folder: llama.cpp
@@ -116,7 +116,7 @@ jobs:
      - name: ccache-buckets-restore
        uses: ./.github/actions/ccache-buckets
        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
        with:
          key: cuda-ubuntu-22.04-hip
          folder: llama.cpp
@@ -167,7 +167,7 @@ jobs:
      - name: ccache-buckets-restore
        uses: ./.github/actions/ccache-buckets
        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
        with:
          key: cuda-ubuntu-22.04-musa
          folder: llama.cpp
@@ -32,8 +32,8 @@ env:
  LLAMA_ARG_LOG_COLORS: 1
  LLAMA_ARG_LOG_PREFIX: 1
  LLAMA_ARG_LOG_TIMESTAMPS: 1
-  # TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback`
-  CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-rollback"
+  # TODO: fix failing tests on OpenVINO backend
+  CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state"

 jobs:
  ubuntu-24-openvino:
@@ -78,8 +78,16 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: sycl-ubuntu-24-${{ matrix.build }}
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: sycl-ubuntu-24-${{ matrix.build }}
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache

      - name: Build
        id: cmake_build
@@ -96,15 +104,17 @@ jobs:
            -DGGML_SYCL_F16=${{ matrix.fp16 }}
          time cmake --build build --config Release -j $(nproc)

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
        env:
-          GH_TOKEN: ${{ github.token }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
        with:
          key: sycl-ubuntu-24-${{ matrix.build }}
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true

  windows-latest-sycl:
    runs-on: windows-2022
@@ -57,8 +57,16 @@ jobs:
        with:
          key: vulkan-ubuntu-24.04-arm
          variant: ccache
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: vulkan-ubuntu-24.04-arm
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache

      - name: Configure
        id: cmake_configure
@@ -73,15 +81,17 @@ jobs:
        run: |
          time cmake --build build -j $(nproc)

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
        env:
-          GH_TOKEN: ${{ github.token }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
        with:
          key: vulkan-ubuntu-24.04-arm
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true

  ubuntu-llvmpipe:
    runs-on: ubuntu-24.04
@@ -115,8 +125,16 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: vulkan-ubuntu-24.04-llvmpipe
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: vulkan-ubuntu-24.04-llvmpipe
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache

      - name: Build
        id: cmake_build
@@ -127,6 +145,18 @@ jobs:
            -DGGML_VULKAN=ON
          cmake --build build --config Release -j $(nproc)

+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: vulkan-ubuntu-24.04-llvmpipe
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
      - name: Test
        id: cmake_test
        run: |
@@ -138,16 +168,6 @@ jobs:
          # test-backend-ops is too slow on llvmpipe, skip it
          ctest -L main -E test-backend-ops --verbose --timeout 900

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: vulkan-ubuntu-24.04-llvmpipe
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
-
  windows:
    runs-on: windows-2025

@@ -57,8 +57,7 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: webgpu-ubuntu-24.04-arm-wasm
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false

      - name: Install Emscripten
        run: |
@@ -76,6 +75,15 @@ jobs:
            "https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}"
          unzip emdawn.zip

+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: webgpu-ubuntu-24.04-arm-wasm
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache
+
      - name: Build WASM WebGPU
        run: |
          source emsdk/emsdk_env.sh
@@ -89,12 +97,14 @@ jobs:

          time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
        env:
-          GH_TOKEN: ${{ github.token }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
        with:
          key: webgpu-ubuntu-24.04-arm-wasm
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
@@ -72,8 +72,7 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: webgpu-macos-latest
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false

      - name: Dawn Dependency
        id: dawn-depends
@@ -88,6 +87,15 @@ jobs:
          mkdir dawn
          tar -xvf artifact.tar.gz -C dawn --strip-components=1

+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: webgpu-macos-latest
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache
+
      - name: Build
        id: cmake_build
        run: |
@@ -95,22 +103,24 @@ jobs:
          cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
          time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)

+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: webgpu-macos-latest
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
      - name: Test
        id: cmake_test
        run: |
          cd build
          ctest -L main --verbose --timeout 900

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: webgpu-macos-latest
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
-
  ubuntu:
    runs-on: ubuntu-24.04

@@ -123,8 +133,7 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: webgpu-ubuntu-24.04
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false

      - name: Dependencies
        id: depends
@@ -148,6 +157,15 @@ jobs:
          mkdir dawn
          tar -xvf artifact.tar.gz -C dawn --strip-components=1

+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: webgpu-ubuntu-24.04
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache
+
      - name: Build
        id: cmake_build
        run: |
@@ -156,6 +174,18 @@ jobs:
            -DGGML_WEBGPU=ON
          time cmake --build build --config Release -j $(nproc)

+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: webgpu-ubuntu-24.04
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
      - name: Test
        id: cmake_test
        run: |
@@ -163,13 +193,3 @@ jobs:
          # This is using llvmpipe and runs slower than other backends
          # test-backend-ops is too slow on llvmpipe, skip it
          ctest -L main -E test-backend-ops --verbose --timeout 900
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: webgpu-ubuntu-24.04
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
@@ -49,14 +49,22 @@ jobs:
        id: depends
        run: |
          sudo apt-get update
-          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3
+          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3 python3-venv python3-pip jq

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: hip-quality-check-ubuntu-22.04
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: hip-quality-check-ubuntu-22.04
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache

      - name: Build with Werror
        id: cmake_build
@@ -85,12 +93,14 @@ jobs:
          make -j $(nproc) 2>&1 | tee metrics.log | grep -v 'Rpass-analysis=kernel-resource-usage\|remark:\|^$'
          python3 ../scripts/hip/gcn-cdna-vgpr-check.py metrics.log

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
        env:
-          GH_TOKEN: ${{ github.token }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
        with:
          key: hip-quality-check-ubuntu-22.04
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
@@ -19,6 +19,7 @@ env:

 permissions:
  contents: write
+  packages: write

 jobs:
  make-release:
@@ -113,6 +114,29 @@ jobs:
              data: await fs.readFileSync('./nightly-tag.txt')
            });

+      - name: Re-tag container images with release version
+        if: ${{ github.event.inputs.dry_run == 'false' && steps.desc.outputs.nightly_tag != '' }}
+        env:
+          GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
+        run: |
+          VERSION="${{ steps.checks.outputs.version }}"
+          NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}"
+          REPO_OWNER="${GITHUB_REPOSITORY_OWNER,,}"
+          IMAGE_REPO="ghcr.io/${REPO_OWNER}/${{ github.event.repository.name }}"
+
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+          VARIANTS=("" "-cuda" "-cuda13" "-vulkan" "-rocm" "-intel" "-musa" "-openvino")
+          TYPES=("full" "light" "server")
+          for type in "${TYPES[@]}"; do
+            for variant in "${VARIANTS[@]}"; do
+              src="${IMAGE_REPO}:${type}${variant}-${NIGHTLY_TAG}"
+              dst="${IMAGE_REPO}:${type}${variant}-${VERSION}"
+              echo "Tagging ${src} -> ${dst}"
+              docker buildx imagetools create --tag "${dst}" "${src}"
+            done
+          done
+
      - name: Dry run summary
        if: ${{ github.event.inputs.dry_run == 'true' }}
        run: |
@@ -103,7 +103,7 @@ jobs:
          source .venv/bin/activate
          cd tools/server/tests
          export ${{ matrix.extra_args }}
-          ./tests.sh
+          PYTEST_WORKERS=1 ./tests.sh

      - name: Slow tests
        id: server_integration_tests_slow
@@ -112,4 +112,4 @@ jobs:
          source .venv/bin/activate
          cd tools/server/tests
          export ${{ matrix.extra_args }}
-          SLOW_TESTS=1 ./tests.sh
+          PYTEST_WORKERS=1 SLOW_TESTS=1 ./tests.sh
@@ -83,8 +83,16 @@ jobs:
        uses: ggml-org/ccache-action@v1.2.24
        with:
          key: server-ubuntu-24.04-arm
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: server-ubuntu-24.04-arm
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache

      - name: Build
        id: cmake_build
@@ -93,6 +101,18 @@ jobs:
            -DGGML_SCHED_NO_REALLOC=ON
          cmake --build build --config Release -j $(nproc) --target llama-server

+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: server-ubuntu-24.04-arm
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
      - name: Python setup
        id: setup_python
        uses: actions/setup-python@v6
@@ -128,16 +148,6 @@ jobs:
          export LLAMA_ARG_BACKEND_SAMPLING=1
          SLOW_TESTS=1 ./tests.sh

-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: server-ubuntu-24.04-arm
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
-
  windows:
    runs-on: windows-2025

@@ -1,4 +1,4 @@
-# date: Tue Aug 18 14:32:43 EEST 2026
+# date: Fri Sep  4 10:06:46 EEST 2026
 # this file is auto-generated by scripts/gen-authors.sh

 Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
@@ -46,6 +46,7 @@ Abhijit Ramesh <abhijitramesh2k@gmail.com>
 abhijitb11 <113058133+abhijitb11@users.noreply.github.com>
 Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
 Abhinay Krishna <abhinaykrishna60@gmail.com>
+Abhiram <78226909+geckguy@users.noreply.github.com>
 Abhishek Gopinath K <31348521+overtunned@users.noreply.github.com>
 abotsis <github@bots.is>
 Abraham Gonzalez <theabecaster0@gmail.com>
@@ -87,6 +88,7 @@ akleine <alb.kleine@gmx.de>
 Al G <toasting@gmail.com>
 Al Mochkin <14274697+amochkin@users.noreply.github.com>
 Alan Gray <agray3@users.noreply.github.com>
+Alan Tseng <alanhc.tseng1999@gmail.com>
 Alawode Oluwandabira <dabiraalawode@yahoo.com>
 Albert Jin <albert.jin@gmail.com>
 Alberto <57916483+albbus-stack@users.noreply.github.com>
@@ -136,7 +138,9 @@ alonfaraj <alonfaraj@gmail.com>
 AlpinDale <52078762+AlpinDale@users.noreply.github.com>
 alwqx <kenan3015@gmail.com>
 Aman <amangupta052@gmail.com>
+Aman Chadha(IVIXMMI) <79802170+ac-mmi@users.noreply.github.com>
 Aman Gupta <amangupta052@gmail.com>
+Aman Karki <itsamankarki@gmail.com>
 amd-dwang <dong.wang@amd.com>
 amd-lalithnc <lalithnc@amd.com>
 Amir <amir_zia@outlook.com>
@@ -187,6 +191,7 @@ Anton Mitkov <anton.mitkov@codeplay.com>
 Antonis Makropoulos <benuix@gmail.com>
 Anudit Nagar <nagaranudit@gmail.com>
 Anuj Attri <anujattri01@gmail.com>
+anujj <ajalota@nvidia.com>
 anzz1 <anzz1@live.com>
 Aparna M P <aparmp@qti.qualcomm.com>
 Aparna M P <quic_aparmp@quicinc.com>
@@ -196,6 +201,7 @@ arch-btw <57669023+arch-btw@users.noreply.github.com>
 arcrank <arcrank@gmail.com>
 ardfork <134447697+ardfork@users.noreply.github.com>
 Arik Poznanski <arikpoz@users.noreply.github.com>
+Aritro Bandyopadhyay <71339004+AriBandyo@users.noreply.github.com>
 arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
 Armen Kaleshian <kriation@users.noreply.github.com>
 Arsen Arutunan <58118221+limloop@users.noreply.github.com>
@@ -230,6 +236,7 @@ bandoti <141645996+bandoti@users.noreply.github.com>
 Bar Haim <barvhaim@gmail.com>
 BarfingLemurs <128182951+BarfingLemurs@users.noreply.github.com>
 Bart Louwers <bart.louwers@gmail.com>
+Bartosz Taudul <wolf@nereid.pl>
 Bartowski <3266127+bartowski1182@users.noreply.github.com>
 Bartowski <ckealty1182@gmail.com>
 Bas Nijholt <basnijholt@gmail.com>
@@ -277,6 +284,7 @@ Bono Lv <lvscar@users.noreply.github.com>
 Borislav Stanimirov <b.stanimirov@abv.bg>
 Borislav Stanimirov <b@ibob.bg>
 Bowen Han <fancycode@gmail.com>
+Brad Smith <1472326+infinitewarp@users.noreply.github.com>
 Branden Butler <bwtbutler@hotmail.com>
 Brandon Squizzato <35474886+bsquizz@users.noreply.github.com>
 Brian <mofosyne@gmail.com>
@@ -287,6 +295,7 @@ Bryan Honof <bryanhonof@gmail.com>
 bryanSwk <93190252+bryanSwk@users.noreply.github.com>
 bsilvereagle <bsilvereagle@users.noreply.github.com>
 bssrdf <merlintiger@hotmail.com>
+Buğra Özgürsoy <13810383+ozgursoy@users.noreply.github.com>
 byte-6174 <88070277+byte-6174@users.noreply.github.com>
 Caleb DeLeeuw <143902425+SolshineCode@users.noreply.github.com>
 Calvin Laurenson <calvin@laurenson.dev>
@@ -326,6 +335,7 @@ Chenguang Li <757486878@qq.com>
 Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
 Chipmunk <101038159+CHIPMUNK-T0T@users.noreply.github.com>
 chiranko <96988916+chiranko@users.noreply.github.com>
+Chris Danis <cdanis@gmail.com>
 Chris Elrod <elrodc@gmail.com>
 Chris Kuehl <ckuehl@ckuehl.me>
 Chris Lee <clee@mg8.org>
@@ -356,6 +366,7 @@ clyang <clyang@clyang.net>
 cmdr2 <secondary.cmdr2@gmail.com>
 cmdr2 <shashank.shekhar.global@gmail.com>
 cocktailpeanut <121128867+cocktailpeanut@users.noreply.github.com>
+codemonkey <441345965@qq.com>
 codezjx <code.zjx@gmail.com>
 coezbek <c.oezbek@gmail.com>
 comex <comexk@gmail.com>
@@ -367,6 +378,8 @@ Copilot <198982749+Copilot@users.noreply.github.com>
 Corentin REGAL <corentin.regal@gmail.com>
 cphlipot <9103367+cphlipot@users.noreply.github.com>
 cpumaxx <163466046+cpumaxx@users.noreply.github.com>
+cqderek <cqderek@gmail.com>
+cqderek <cqiang@qti.qualcomm.com>
 crasm <crasm@git.vczf.net>
 crasm <crasm@git.vczf.us>
 crat0z <11581854+crat0z@users.noreply.github.com>
@@ -427,6 +440,7 @@ DavidKorczynski <david@adalogics.com>
 davidrhodus <david@vacovideo.com>
 Dawid Potocki <github@dawidpotocki.com>
 Dawid Wysocki <62249621+TortillaZHawaii@users.noreply.github.com>
+Daya Adianto <addianto@users.noreply.github.com>
 ddh0 <chemist-mulches-39@icloud.com>
 ddh0 <dylanhalladay02@icloud.com>
 ddpasa <112642920+ddpasa@users.noreply.github.com>
@@ -463,6 +477,7 @@ Dmytro Romanov <casteldazur@gmail.com>
 Dobri Danchev <12420863+danchev@users.noreply.github.com>
 DocShotgun <126566557+DocShotgun@users.noreply.github.com>
 Doctor Shotgun <126566557+DocShotgun@users.noreply.github.com>
+Dominik Pantaleoni <95251853+dpantaleoni@users.noreply.github.com>
 Don Mahurin <dmahurin@users.noreply.github.com>
 Dong Won Kim <63934649+ddwkim@users.noreply.github.com>
 Donghyeon Jeong <54725479+djeong20@users.noreply.github.com>
@@ -504,6 +519,7 @@ Emmanuel Ferdman <emmanuelferdman@gmail.com>
 Emreerdog <34742675+Emreerdog@users.noreply.github.com>
 Engininja2 <139037756+Engininja2@users.noreply.github.com>
 Equim <sayaka@ekyu.moe>
+Eric A Stalee <87948564+Eric-A-Stalee@users.noreply.github.com>
 Eric Curtin <ecurtin@redhat.com>
 Eric Curtin <eric.curtin@docker.com>
 Eric Curtin <ericcurtin17@gmail.com>
@@ -519,6 +535,7 @@ Esko Toivonen <eskot98@gmail.com>
 Ethan Turner <eturner64@gmail.com>
 Ettore Di Giacinto <mudler@users.noreply.github.com>
 EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com>
+Eurekatic <eurekatic@eurekatic.eu>
 Evan Huus <eapache@gmail.com>
 Evan Jones <evan.q.jones@gmail.com>
 Evan Miller <emmiller@gmail.com>
@@ -677,6 +694,7 @@ HimariO <dsfhe49854@gmail.com>
 hipudding <huafengchun@gmail.com>
 Hitesh Chopra <34310832+hiteshchopra11@users.noreply.github.com>
 hksdpc255 <43977088+hksdpc255@users.noreply.github.com>
+hmirin <hmirin@users.noreply.github.com>
 hmscider <201289679+hmscider@users.noreply.github.com>
 Hoang Nguyen <hugo53@users.noreply.github.com>
 hoangmit <hoangmit@users.noreply.github.com>
@@ -701,6 +719,7 @@ Huawei Lin <huaweilin.cs@gmail.com>
 Hugo <hugo@whynothugo.nl>
 Hugo Roussel <hugo.rous@gmail.com>
 Huifeng Ou <79071290+ho2103@users.noreply.github.com>
+HumerousGorgon <31957201+HumerousGorgon@users.noreply.github.com>
 hutli <6594598+hutli@users.noreply.github.com>
 hutli <hutli@hutli.hu>
 hutli <jensstaermose@hotmail.com>
@@ -738,12 +757,15 @@ intelmatt <61025942+intelmatt@users.noreply.github.com>
 iohub <rickyang.pro@gmail.com>
 Ionoclast Laboratories <brigham@ionoclast.com>
 iron <lizhenneng@gmail.com>
+Isaac <34376531+init-22@users.noreply.github.com>
 Isaac McFadyen <isaac@imcf.me>
 IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
 Ishaan Gandhi <Ishaangandhi@gmail.com>
 iSma <ismail.senhaji@gmail.com>
 Ismail <115064057+AlrIsmail@users.noreply.github.com>
 issixx <46835150+issixx@users.noreply.github.com>
+itsnotoger <19309683+itsnotoger@users.noreply.github.com>
+itterative <190138728+itterative@users.noreply.github.com>
 Ivan <nekotekina@gmail.com>
 Ivan Chikish <nekotekina@gmail.com>
 Ivan Filipov <159561759+vanaka11@users.noreply.github.com>
@@ -768,6 +790,7 @@ Jakkala Mahesh <155058658+MaheshJakkala@users.noreply.github.com>
 Jakub N <jakubniemczyk97@gmail.com>
 JamePeng <jame_peng@sina.com>
 James A Capozzoli <157492257+jac-jim@users.noreply.github.com>
+James Francis <6763899+JamesFranc@users.noreply.github.com>
 James O'Leary <65884233+jpohhhh@users.noreply.github.com>
 James Reynolds <magnusviri@users.noreply.github.com>
 jameswu2014 <545426914@qq.com>
@@ -798,6 +821,7 @@ Jed Fox <git@jedfox.com>
 Jeff Bolz <jbolz@nvidia.com>
 Jeffrey Morgan <jmorganca@gmail.com>
 Jeffrey Quesnelle <emozilla@nousresearch.com>
+Jeremie Miller <jeremie.miller@gmail.com>
 Jeremy Demeule <jdemeule@users.noreply.github.com>
 Jeremy Rand <244188+JeremyRand@users.noreply.github.com>
 Jeroen Mostert <jeroen.mostert@cm.com>
@@ -809,6 +833,7 @@ Jesse Jojo Johnson <williamsaintgeorge@gmail.com>
 Jesse LaRose <jesse@taey.ai>
 Jesse Posner <jesse.posner@gmail.com>
 Jesus Talavera <145992175+jesus-talavera-ibm@users.noreply.github.com>
+Jetson Tan <tanzongyouyi@outlook.com>
 Jett Janiak <jettjaniak@gmail.com>
 Jeximo <jeximo@gmail.com>
 JFLFY2255 <JFLFY2255@163.com>
@@ -825,6 +850,7 @@ Jie Fu (傅杰) <jiefu@tencent.com>
 jiez <373447296@qq.com>
 Jillis ter Hove <j.terhove@gmail.com>
 Jim Wu <jimw567@users.noreply.github.com>
+Jingxin (Philip) Li <philipaslee@gmail.com>
 Jinwoo Jeong <33892306+williamjeong2@users.noreply.github.com>
 Jinyang He <hejinyang@loongson.cn>
 jinzihao <jinzihao1996@gmail.com>
@@ -850,11 +876,13 @@ John Balis <phobossystems@gmail.com>
 John Bean <113509988+johnbean393@users.noreply.github.com>
 John Eismeier <42679190+jeis4wpi@users.noreply.github.com>
 John Smith <67539080+kingsidelee@users.noreply.github.com>
+John-Henry Lim <42513874+Interpause@users.noreply.github.com>
 Johnathan Craig Maudlin <13183098+jcmdln@users.noreply.github.com>
 JohnnyB <jboero@users.noreply.github.com>
 johnson442 <56517414+johnson442@users.noreply.github.com>
 jojorne <jojorne@users.noreply.github.com>
 jon-chuang <9093549+jon-chuang@users.noreply.github.com>
+Jonas J <111707981+John-194@users.noreply.github.com>
 Jonas Jankaitis <111707981+John-194@users.noreply.github.com>
 Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
 Jonathan <47618606+jbuchananr@users.noreply.github.com>
@@ -924,6 +952,7 @@ Karsten Weiss <knweiss@gmail.com>
 Karthick <j.karthic2004@gmail.com>
 Karthik Kumar Viswanathan <195178+guilt@users.noreply.github.com>
 Karthik Sethuraman <k.seth1993@gmail.com>
+Kartik Gulia <kgulia@nvidia.com>
 Kartik Sirohi <99896785+sirohikartik@users.noreply.github.com>
 Kashif Rasul <kashif.rasul@gmail.com>
 KASR <karim.asrih@gmail.com>
@@ -931,6 +960,7 @@ Kasumi <90275229+kasumi-1@users.noreply.github.com>
 Katostrofik <georgiopapairo@gmail.com>
 katsu560 <118887472+katsu560@users.noreply.github.com>
 Kawrakow <48489457+ikawrakow@users.noreply.github.com>
+kbenkhaled <khalilbenkhaled01@gmail.com>
 kchro3 <62481661+kchro3@users.noreply.github.com>
 kdkd <2569413+kdkd@users.noreply.github.com>
 Keiichi Tabata <keiichi.tabata@outlook.com>
@@ -939,6 +969,7 @@ Kenvix ⭐ <kenvixzure@live.com>
 Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com>
 Kevin Gibbons <bakkot@gmail.com>
 Kevin Hannon <kehannon@redhat.com>
+Kevin Hopper <93635715+kh0pper@users.noreply.github.com>
 Kevin Ji <1146876+kevinji@users.noreply.github.com>
 Kevin Kwok <antimatter15@gmail.com>
 Kevin Liu <4396kevinliu@gmail.com>
@@ -964,12 +995,14 @@ Konstantin Herud <konstantin.herud@denkbares.com>
 Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
 Krishna Sridhar <99914379+srikris-sridhar@users.noreply.github.com>
 krystiancha <krystian@krystianch.com>
+krzsztf <krzysztof@witkowscy.org>
 kubawoo <k-wach@o2.pl>
 kumaal <44551860+kumaal@users.noreply.github.com>
 kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com>
 kunnis <kunnis@users.noreply.github.com>
 Kunshang Ji <kunshang.ji@intel.com>
 kuronekosaiko <EvanChanJ@163.com>
+kurquhar <kurquhar@qti.qualcomm.com>
 Kusha Gharahi <3326002+kushagharahi@users.noreply.github.com>
 kustaaya <58045274+kustaaya@users.noreply.github.com>
 kuvaus <22169537+kuvaus@users.noreply.github.com>
@@ -981,6 +1014,7 @@ Kyle Liang <liangmanlai@gmail.com>
 Kyle Mistele <kyle@mistele.com>
 KyleHagy <59183061+KyleHagy@users.noreply.github.com>
 Kylin <56434533+KyL0N@users.noreply.github.com>
+Kyozzz <1147385157@qq.com>
 l-austenfeld <53152202+l-austenfeld@users.noreply.github.com>
 l3utterfly <gc.pthzfoldr@gmail.com>
 l8bloom <l8bloomapi@gmail.com>
@@ -992,6 +1026,7 @@ Lars Sonchocky-Helldorf <lars.sonchocky-helldorf@hamburg.de>
 las7 <98077186+las7@users.noreply.github.com>
 Lasse Lauwerys <65569591+Iemand005@users.noreply.github.com>
 Laura <Tijntje_7@msn.com>
+Laurent Zuijdwijk <laurent.zuijdwijk@gmail.com>
 Law Po Ying <30721578+yingying0906@users.noreply.github.com>
 lcy <lcy0321@users.noreply.github.com>
 ldwang <ftgreat@163.com>
@@ -1039,6 +1074,8 @@ Ludovic Henry <git@ludovic.dev>
 Ludovic Henry <ludovic@rivosinc.com>
 Lukas Straub <lukasstraub2@web.de>
 Łukasz Ślusarczyk <112692748+lslusarczyk@users.noreply.github.com>
+Lukasz Stolcman <4583553+lstolcman@users.noreply.github.com>
+LunalFresh <165352784+LunalFresh@users.noreply.github.com>
 Luo Tian <lt@basecity.com>
 luoyu-intel <yu.luo@intel.com>
 luyhcsu <110711054+luyhcsu@users.noreply.github.com>
@@ -1054,6 +1091,7 @@ Maarten ter Huurne <maarten@treewalker.org>
 Maciej Lisowski <39798354+MaciejDromin@users.noreply.github.com>
 Mack Straight <eiz@users.noreply.github.com>
 maddes8cht <55592906+maddes8cht@users.noreply.github.com>
+Mads Marquart <mads@marquart.dk>
 Maël Kerbiriou <m431.kerbiriou@gmail.com>
 MaggotHATE <clay1326@gmail.com>
 MagicExists <106458387+gugugiyu@users.noreply.github.com>
@@ -1215,6 +1253,8 @@ Naco Siren <naco-siren@users.noreply.github.com>
 Nam D. Tran <42194884+namtranase@users.noreply.github.com>
 nanahi <130121847+na-na-hi@users.noreply.github.com>
 Nathan Epstein <nate2@umbc.edu>
+Nathan Wilson <67372905+Nathanw1014@users.noreply.github.com>
+Nathanw1014 <67372905+Nathanw1014@users.noreply.github.com>
 Natsu <chino@hotococoa.moe>
 Nauful Shaikh <nauful@gmail.com>
 NawafAlansari <72708095+NawafAlansari@users.noreply.github.com>
@@ -1237,6 +1277,7 @@ niansa/tuxifan <tuxifan@posteo.de>
 Nicholai Tukanov <nicholaitukanov@gmail.com>
 Nicholas Sparks <157740354+nisparks@users.noreply.github.com>
 Nick <0x0b4ac@gmail.com>
+Nick Farrell <nick.farrell@aiven.io>
 nick huang <nickhuang99@hotmail.com>
 Nick Lafleur <55208706+nicklafleur@users.noreply.github.com>
 Nick Towle <ntowle@gmail.com>
@@ -1259,6 +1300,7 @@ NikolaiLyssogor <59844691+NikolaiLyssogor@users.noreply.github.com>
 Nikolaos Pothitos <pothitos@di.uoa.gr>
 Nikolas <127742645+nneubacher@users.noreply.github.com>
 Nikolay Popov <131475237+npopov-vst@users.noreply.github.com>
+Nils Gladitz <nilsgladitz@gmail.com>
 Nindaleth <Nindaleth@users.noreply.github.com>
 ningshanwutuobang <ningshanwutuobang@gmail.com>
 Noah <99681487+NoahOksuz@users.noreply.github.com>
@@ -1355,6 +1397,7 @@ Pop Flamingo <trevor.annedenise@icloud.com>
 postmasters <namnguyen@google.com>
 Pouya <PooyaGhahramanian@Gmail.com>
 pqnet <119850+pqnet@users.noreply.github.com>
+Prabhsimran Singh <pskrunner14@gmail.com>
 Prabod <prabod@maincode.com>
 Prajwal B Mehendarkar <prajwal.b.mehendarkar@ibm.com>
 Pranav Dhinakar <pdhinaka@qti.qualcomm.com>
@@ -1378,6 +1421,7 @@ qouoq <qouoq@fastmail.com>
 Qu Zongfu <43257352+yancaoweidaode@users.noreply.github.com>
 quei <56998528+quei4r@users.noreply.github.com>
 Quentin Bramas <quentin.bramas@gmail.com>
+QuintinShaw <github@xyt.email>
 QuintinShaw <yx6f20@soton.ac.uk>
 qunash <anzoria@gmail.com>
 quyentonndbs <raynaedgar8677@outlook.com>
@@ -1462,6 +1506,7 @@ robertomeroni <150194833+robertomeroni@users.noreply.github.com>
 Robey Holderith <robey@flaminglunchbox.net>
 Robin Davidsson <40024429+R-Dson@users.noreply.github.com>
 Robyn <robyngraf@users.noreply.github.com>
+Rock Chen <rockchen.tw@gmail.com>
 Rőczey Barnabás <31726601+An0nie@users.noreply.github.com>
 RodriMora <bullerwins@gmail.com>
 Roger Chen <chenrui@gmail.com>
@@ -1499,17 +1544,21 @@ runfuture <runfuture@users.noreply.github.com>
 RunningLeon <maningsheng@sensetime.com>
 RunningLeon <mnsheng@yeah.net>
 Russyyds <161207317+Russyyds@users.noreply.github.com>
+Ryan C <ryan5rdx@users.noreply.github.com>
 Ryan Goulden <percontation@gmail.com>
 Ryan Landay <rlanday@gmail.com>
 Ryan Mangeno <160974989+ryan-mangeno@users.noreply.github.com>
 Ryder Wishart <ryderwishart@gmail.com>
 Ryuei <louixs@users.noreply.github.com>
 s-goto-11 <206795233+s-goto-11@users.noreply.github.com>
+s0mecode <213953308+s0mecode@users.noreply.github.com>
 s8322 <s0527684199@gmail.com>
+Saad Ali <NIXKnight@users.noreply.github.com>
 Saba Fallah <10401143+sfallah@users.noreply.github.com>
 Saba Fallah <sabafallah@gmail.com>
 Sachin Desai <smdesai@gmail.com>
 Sachin Sharma <sachin@zettabolt.com>
+Safi Ullah <safiullah.3915@gmail.com>
 safranowith <bsh155762@gmail.com>
 SakuraUmi <yukinon244@gmail.com>
 Salvador E. Tropea <stropea@inti.gob.ar>
@@ -1552,6 +1601,7 @@ Sergey Alirzaev <l29ah@riseup.net>
 Sergey Alirzaev <zl29ah@gmail.com>
 Sergey Fedorov <vital.had@gmail.com>
 Sergey Malinin <sergmalinin@gmail.com>
+Sergey Sklyarov <sergey.sklyarov@gmail.com>
 Sergio López <slp@redhat.com>
 Sergio López <slp@sinrega.org>
 Sergiu <8598216+mzsergiu@users.noreply.github.com>
@@ -1582,11 +1632,13 @@ Shawn Gu <shawngu@qti.qualcomm.com>
 Shawn yang <137684499+Yangxiaoz@users.noreply.github.com>
 Shelby Jenkins <47464908+ShelbyJenkins@users.noreply.github.com>
 Sheldon Robinson <sheldon.robinson@live.com>
+Shenghan Yang <ysharke@sjtu.edu.cn>
 shibe2 <shibe@tuta.io>
 Shijie <821898965@qq.com>
 Shin-myoung-serp <relent95@naver.com>
 Shintarou Okada <kokuzen@gmail.com>
 shivamkumard-ctrl <shivamkumard@nvidia.com>
+Shobhit <sobhit.me@gmail.com>
 Shouyu <65317431+joeldushouyu@users.noreply.github.com>
 Shouzheng Liu <61452103+lshzh-ww@users.noreply.github.com>
 Shouzheng Liu <lshzh.hi@gmail.com>
@@ -1607,6 +1659,7 @@ Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
 Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
 simevo <github@simevo.com>
 Simon Redman <simon@ergotech.com>
+Simon Teixidor <simon@flaskpost.me>
 Simon Willison <swillison@gmail.com>
 simon886212 <37953122+simon886212@users.noreply.github.com>
 Simranjeet Singh <105192966+simrnsingh@users.noreply.github.com>
@@ -1663,6 +1716,7 @@ stevenkuang <stevenkuang@tencent.com>
 Steward Garcia <57494570+FSSRepo@users.noreply.github.com>
 StrangeBytesDev <141275258+StrangeBytesDev@users.noreply.github.com>
 strawberrymelonpanda <152940198+strawberrymelonpanda@users.noreply.github.com>
+Strongtut <Strongtut@users.noreply.github.com>
 Suaj Carrot <72162667+SuajCarrot@users.noreply.github.com>
 sudhiarm <sudhi.sathyavathy@arm.com>
 Sukriti Sharma <Ssukriti@users.noreply.github.com>
@@ -1687,6 +1741,7 @@ Tamar <Tamar0812@outlook.co.il>
 tamarPal <tamarp3385@gmail.com>
 Tameem <113388789+AhmadTameem@users.noreply.github.com>
 Tamotsu Takahashi <ttakah+github@gmail.com>
+Tanner Bruhn <66120666+tannerbruhn@users.noreply.github.com>
 tarcey <cey.tarik@gmail.com>
 Tarek Dakhran <t.dakhran@gmail.com>
 Tarek Dakhran <tarek@liquid.ai>
@@ -1696,6 +1751,7 @@ Taylor <quantumtraveling@gmail.com>
 tc-mb <157115220+tc-mb@users.noreply.github.com>
 TecJesh <qdvm5gl@163.com>
 Tei Home <taiteitonghome@proton.me>
+Tekin Ertekin <tekin.ertekin@gmail.com>
 Tekin Ertekin <tekinertekin@gmail.com>
 tempstudio <49735574+tempstudio@users.noreply.github.com>
 teo <TeoZosa@users.noreply.github.com>
@@ -1737,6 +1793,7 @@ Ting Lou <louting@189.cn>
 Ting Lou <ting.lou@gmail.com>
 Ting Sun <suntcrick@gmail.com>
 Titaniumtown <titaniumtown@proton.me>
+Tiwei Bie <tiwei.btw@antgroup.com>
 tjohnman <tjohnman@users.noreply.github.com>
 Tobias Lütke <tobi@shopify.com>
 Toby <25832191+aetherbird@users.noreply.github.com>
@@ -1813,6 +1870,7 @@ Vishal Agarwal <vishalagarwal.jss@gmail.com>
 Vishal Singh <vishal@zettabolt.com>
 Vitali Lovich <vlovich+github@gmail.com>
 Vivian <vynride@gmail.com>
+vk <89937361+itsvedantkumar@users.noreply.github.com>
 Vlad <spitfireage@gmail.com>
 Vladimir <bogdad@gmail.com>
 Vladimir Malyutin <first-leon@yandex.ru>
@@ -1897,6 +1955,7 @@ Yaiko <elyaiko@hotmail.com>
 Yakine Tahtah <96926916+ReinforcedKnowledge@users.noreply.github.com>
 YangLe <smilingpoplar@gmail.com>
 yangli2 <yangli2@gmail.com>
+Yaniss Amazouz <yaniss91600@gmail.com>
 Yann Follet <131855179+YannFollet@users.noreply.github.com>
 Yanzhao Wang <yanzhaow@qti.qualcomm.com>
 Yarden Tal <yardent@qti.qualcomm.com>
@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)

 ### llama.cpp version
 set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 3)
+set(LLAMA_VERSION_MINOR 4)
 set(LLAMA_VERSION_PATCH 0)
 set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")

@@ -13,7 +13,7 @@
 [![Docker](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/docker.yml?label=Docker)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
 [![Winget](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/winget.yml?label=Winget)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)

-[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
+[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)

 </div>

@@ -74,7 +74,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
 | [CANN](docs/build.md#cann) | Ascend NPU |
 | [CUDA](docs/build.md#cuda) | Nvidia GPU |
 | [HIP](docs/build.md#hip) | AMD GPU |
-| [Hexagon [In Progress]](docs/backend/snapdragon/README.md) | Snapdragon |
+| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
 | [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
 | [MUSA](docs/build.md#musa) | Moore Threads GPU |
 | [Metal](docs/build.md#metal-build) | Apple Silicon |
@@ -80,7 +80,7 @@ static const command cmds[] = {
 #undef UPDATE_HIDDEN

 static int version(int /*argc*/, char ** /*argv*/) {
-    llama_print_build_info(llama_version());
+    llama_print_build_info(llama_version(), stdout);
    return 0;
 }

@@ -189,8 +189,8 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
    fi
    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"

-    # TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback*`
-    CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-rollback"
+    # TODO: fix failing tests on OpenVINO backend
+    CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state"
 fi

 ## helpers
@@ -3901,6 +3901,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
            common_log_set_file(common_log_main(), value.c_str());
        }
    ).set_env("LLAMA_ARG_LOG_FILE"));
+    add_opt(common_arg(
+        {"--log-jsonl"},
+        {"--no-log-jsonl"},
+        "Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)",
+        [](common_params &, bool value) {
+            common_log_set_jsonl(common_log_main(), value);
+        }
+    ).set_env("LLAMA_ARG_LOG_JSONL"));
    add_opt(common_arg(
        {"--log-prompts-dir"}, "PATH",
        "Log prompts to directory (auto-created if not present; only used for debugging, default: disabled)",
@@ -29,7 +29,7 @@ const char * llama_build_info(void) {
    return s.c_str();
 }

-void llama_print_build_info(const char * llama_version) {
-    fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
-    fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
+void llama_print_build_info(const char * llama_version, FILE * stream) {
+    fprintf(stream, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
+    fprintf(stream, "built with %s for %s\n", llama_compiler(), llama_build_target());
 }
@@ -1,5 +1,7 @@
 #pragma once

+#include <cstdio>
+
 int llama_build_number(void);

 const char * llama_commit(void);
@@ -8,4 +10,4 @@ const char * llama_compiler(void);
 const char * llama_build_target(void);
 const char * llama_build_info(void);

-void llama_print_build_info(const char *);
+void llama_print_build_info(const char *, FILE * = stderr);
@@ -117,6 +117,7 @@ caps caps_get(jinja::program & prog) {

    JJ_DEBUG("%s\n", ">>> Running capability check: typed content");

+    bool checks_for_string = false;
    static const std::string content_marker = "STRING_MARKER";

    // case: typed content support
@@ -136,6 +137,10 @@ caps caps_get(jinja::program & prog) {
        [&](context &, bool success, value & messages, value &, const std::string & rendered) {
            auto & content = messages->at(0)->at("content");
            caps_print_stats(content, "messages[0].content");
+            if (has_op(content, "test_is_string")) {
+                // checked if content is string
+                checks_for_string = true;
+            }
            bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");
            if (used_as_array) {
                // accessed as an array
@@ -151,6 +156,33 @@ caps caps_get(jinja::program & prog) {
        }
    );

+    if (checks_for_string) {
+        caps_try_execute(
+            prog,
+            [&]() {
+                // messages
+                return json::array({
+                    {
+                        {"role", "user"},
+                        {"content", json::array({
+                        })}
+                    }
+                });
+            },
+            nullptr, // ctx_fn
+            nullptr, // tools_fn
+            [&](context &, bool success, value & messages, value &, const std::string &) {
+                auto & content = messages->at(0)->at("content");
+                caps_print_stats(content, "messages[0].content");
+                bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");
+                if (used_as_array && success) {
+                    // accessed as an array
+                    result.supports_typed_content = true;
+                }
+            }
+        );
+    }
+
    JJ_DEBUG("%s\n", ">>> Running capability check: system prompt");

    // case: system prompt support
@@ -412,12 +412,18 @@ value test_expression::execute_impl(context & ctx) {
        throw std::runtime_error("Invalid test expression");
    }

-    auto it = builtins.find("test_is_" + test_id);
-    JJ_DEBUG("Test expression %s '%s' %s (using function 'test_is_%s')", operand->type().c_str(), test_id.c_str(), negate ? "(negate)" : "", test_id.c_str());
+    const std::string test_name = "test_is_" + test_id;
+    auto it = builtins.find(test_name);
+    JJ_DEBUG("Test expression %s '%s' %s (using function '%s')", operand->type().c_str(), test_id.c_str(), negate ? "(negate)" : "", test_name.c_str());
    if (it == builtins.end()) {
        throw std::runtime_error("Unknown test '" + test_id + "'");
    }

+    if (ctx.is_get_stats) {
+        value_t::stats_t::mark_used(input);
+        input->stats.ops.insert(test_name);
+    }
+
    auto res = it->second(args);

    if (negate) {
@@ -748,6 +748,10 @@ private:
            optional_props.push_back("*");
        }

+        if (required_props.empty() && optional_props.empty()) {
+            return "\"{\" space \"}\"";
+        }
+
        std::string rule = "\"{\" space ";
        for (size_t i = 0; i < required_props.size(); i++) {
            if (i > 0) {
@@ -1,5 +1,6 @@
 #include "common.h"
 #include "log.h"
+#include "json.h"

 #include <chrono>
 #include <condition_variable>
@@ -66,6 +67,17 @@ static const char* g_col[] = {
    "",
 };

+static const char * level_str(enum ggml_log_level level) {
+    switch (level) {
+        case GGML_LOG_LEVEL_DEBUG: return "debug";
+        case GGML_LOG_LEVEL_INFO:  return "info";
+        case GGML_LOG_LEVEL_WARN:  return "warn";
+        case GGML_LOG_LEVEL_ERROR: return "error";
+        case GGML_LOG_LEVEL_CONT:  return "cont";
+        default:                   return "none";
+    }
+}
+
 struct common_log_entry {
    enum ggml_log_level level {GGML_LOG_LEVEL_INFO};

@@ -74,6 +86,7 @@ struct common_log_entry {
    int64_t timestamp { 0 };
    bool is_end       { false }; // signals the worker thread to stop
    bool prefix       { false };
+    bool jsonl        { false };

    common_log_entry(size_t size = 256) : msg(size) { }

@@ -88,11 +101,23 @@ struct common_log_entry {

            fcur = stdout;

-            if (level != GGML_LOG_LEVEL_NONE) {
+            if (level != GGML_LOG_LEVEL_NONE && !jsonl) {
                fcur = stderr;
            }
        }

+        if (jsonl) {
+            common_json obj = {
+                {"type",  "log"},
+                {"time",  timestamp},
+                {"level", level_str(level)},
+                {"msg",   msg.data()},
+            };
+            fprintf(fcur, "%s\n", obj.dump_safe().c_str());
+            fflush(fcur);
+            return;
+        }
+
        if (level != GGML_LOG_LEVEL_NONE && level != GGML_LOG_LEVEL_CONT && prefix) {
            if (timestamp) {
                // [M.s.ms.us]
@@ -131,6 +156,7 @@ struct common_log {
        file       = nullptr;
        prefix     = false;
        timestamps = false;
+        jsonl      = false;
        running    = false;
        t_start    = t_us();

@@ -158,6 +184,7 @@ private:

    bool prefix;
    bool timestamps;
+    bool jsonl;
    bool running;

    int64_t t_start;
@@ -246,6 +273,7 @@ public:
        entry.is_end    = false;
        entry.level     = level;
        entry.prefix    = prefix;
+        entry.jsonl     = jsonl;
        entry.timestamp = 0;
        if (timestamps) {
            entry.timestamp = t_us() - t_start;
@@ -360,6 +388,12 @@ public:

        this->timestamps = timestamps;
    }
+
+    void set_jsonl(bool jsonl) {
+        std::lock_guard<std::mutex> lock(mtx);
+
+        this->jsonl = jsonl;
+    }
 };

 //
@@ -433,6 +467,10 @@ void common_log_set_timestamps(struct common_log * log, bool timestamps) {
    log->set_timestamps(timestamps);
 }

+void common_log_set_jsonl(struct common_log * log, bool jsonl) {
+    log->set_jsonl(jsonl);
+}
+
 void common_log_flush(struct common_log * log) {
    log->pause();
    log->resume();
@@ -91,6 +91,7 @@ void common_log_set_file      (struct common_log * log, const char * file); // n
 void common_log_set_colors    (struct common_log * log, log_colors colors); // not thread-safe
 void common_log_set_prefix    (struct common_log * log, bool prefix);       // whether to output prefix to each log
 void common_log_set_timestamps(struct common_log * log, bool timestamps);   // whether to output timestamps in the prefix
+void common_log_set_jsonl     (struct common_log * log, bool jsonl);        // print each log as a JSON object on one line, not thread-safe
 void common_log_flush         (struct common_log * log);                    // flush all pending log messages

 // helper macros for logging
@@ -124,6 +124,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
    "HunYuanMoEV1ForCausalLM": "hunyuan",
    "HunYuanVLForConditionalGeneration": "hunyuan",
    "HYV3ForCausalLM": "hunyuan",
+    "HYV4ForCausalLM": "hy_v4",
    "IQuestCoderForCausalLM": "llama",
    "InternLM2ForCausalLM": "internlm",
    "InternLM3ForCausalLM": "internlm",
@@ -254,6 +255,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
    "SeedOssForCausalLM": "olmo",
    "SmallThinkerForCausalLM": "smallthinker",
    "SmolLM3ForCausalLM": "llama",
+    "Spark2_5ForCausalLM": "spark2_5",
    "SolarOpenForCausalLM": "glm",
    "StableLMEpochForCausalLM": "stablelm",
    "StableLmForCausalLM": "stablelm",
@@ -130,7 +130,8 @@ class ModelBase:
                 sentence_transformers_dense_modules: bool = False,
                 target_model_dir: Path | None = None,
                 fuse_gate_up_exps: bool = False,
-                 fp8_as_q8: bool = False):
+                 fp8_as_q8: bool = False,
+                 fuse_qkv: bool = False):
        if type(self) is ModelBase or \
                type(self) is TextModel or \
                type(self) is MmprojModel:
@@ -153,6 +154,15 @@ class ModelBase:
        self.fuse_gate_up_exps = fuse_gate_up_exps
        self._gate_exp_buffer: dict[int, Tensor] = {}
        self._up_exp_buffer: dict[int, Tensor] = {}
+        self.fuse_qkv = fuse_qkv
+        self._q_buffer: dict[int, Tensor] = {}
+        self._k_buffer: dict[int, Tensor] = {}
+        self._v_buffer: dict[int, Tensor] = {}
+        self._q_bias_buffer: dict[int, Tensor] = {}
+        self._k_bias_buffer: dict[int, Tensor] = {}
+        self._v_bias_buffer: dict[int, Tensor] = {}
+        self._fusable_qkv_weight_layers: set[int] = set()
+        self._fusable_qkv_bias_layers: set[int] = set()
        self.hparams = ModelBase.load_hparams(self.dir_model, self.is_mistral_format) if hparams is None else hparams
        self.model_tensors = self.index_tensors(remote_hf_model_id=remote_hf_model_id)
        self.metadata_override = metadata_override
@@ -617,6 +627,43 @@ class ModelBase:
            raise ValueError(f"Can not map tensor {name!r}")
        return new_name

+    def prepare_qkv_fusion(self) -> None:
+        self._fusable_qkv_weight_layers.clear()
+        self._fusable_qkv_bias_layers.clear()
+        if not self.fuse_qkv or gguf.MODEL_TENSOR.ATTN_QKV not in gguf.MODEL_TENSORS[self.model_arch]:
+            return
+
+        qkv_types = {
+            gguf.MODEL_TENSOR.ATTN_Q,
+            gguf.MODEL_TENSOR.ATTN_K,
+            gguf.MODEL_TENSOR.ATTN_V,
+        }
+        weights: dict[int, set[gguf.MODEL_TENSOR]] = {}
+        biases: dict[int, set[gguf.MODEL_TENSOR]] = {}
+
+        for name in self.model_tensors:
+            mapped = self.tensor_map.get_type_and_name(name, try_suffixes=(".weight", ".bias"))
+            if mapped is None:
+                continue
+            tensor_type, new_name = mapped
+            if tensor_type not in qkv_types:
+                continue
+
+            bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None)
+            if bid is None:
+                continue
+            if new_name.endswith(".weight"):
+                weights.setdefault(bid, set()).add(tensor_type)
+            elif new_name.endswith(".bias"):
+                biases.setdefault(bid, set()).add(tensor_type)
+
+        for bid, weight_types in weights.items():
+            bias_types = biases.get(bid, set())
+            if weight_types == qkv_types and (not bias_types or bias_types == qkv_types):
+                self._fusable_qkv_weight_layers.add(bid)
+                if bias_types:
+                    self._fusable_qkv_bias_layers.add(bid)
+
    def set_gguf_parameters(self):
        raise NotImplementedError("set_gguf_parameters() must be implemented in subclasses")

@@ -645,6 +692,40 @@ class ModelBase:
               self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_UP_EXP, bid):
                return []

+        # Handle Q/K/V tensor fusion if enabled
+        qkv_bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None) if self.fuse_qkv else None
+        if qkv_bid is not None:
+            is_bias = new_name.endswith('.bias')
+            suffix = '.bias' if is_bias else '.weight'
+            fusable_layers = self._fusable_qkv_bias_layers if is_bias else self._fusable_qkv_weight_layers
+            if qkv_bid not in fusable_layers:
+                return [(new_name, data_torch)]
+
+            buf_q = self._q_bias_buffer if is_bias else self._q_buffer
+            buf_k = self._k_bias_buffer if is_bias else self._k_buffer
+            buf_v = self._v_bias_buffer if is_bias else self._v_buffer
+
+            if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_Q, qkv_bid, suffix):
+                buf_q[qkv_bid] = data_torch
+            elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_K, qkv_bid, suffix):
+                buf_k[qkv_bid] = data_torch
+            elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_V, qkv_bid, suffix):
+                buf_v[qkv_bid] = data_torch
+
+            if qkv_bid in buf_q and qkv_bid in buf_k and qkv_bid in buf_v:
+                q_data = buf_q.pop(qkv_bid)
+                k_data = buf_k.pop(qkv_bid)
+                v_data = buf_v.pop(qkv_bid)
+                fused_data = torch.cat([q_data, k_data, v_data], dim=0)
+                fused_name = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, qkv_bid, suffix=suffix)
+                logger.info(f"Fused Q, K, V {suffix[1:]} into QKV for layer {qkv_bid}")
+                return [(fused_name, fused_data)]
+
+            if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_Q, qkv_bid, suffix) or \
+               self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_K, qkv_bid, suffix) or \
+               self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_V, qkv_bid, suffix):
+                return []
+
        return [(new_name, data_torch)]

    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
@@ -899,6 +980,8 @@ class ModelBase:

        self.dequant_model()

+        self.prepare_qkv_fusion()
+
        # Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
        if self.tensor_map.mapping:
            max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
@@ -1027,6 +1110,13 @@ class ModelBase:

                self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)

+        qkv_buffers = (
+            self._q_buffer, self._k_buffer, self._v_buffer,
+            self._q_bias_buffer, self._k_bias_buffer, self._v_bias_buffer,
+        )
+        if any(qkv_buffers):
+            raise ValueError("QKV fusion did not consume all buffered tensors")
+
    def set_type(self):
        self.gguf_writer.add_type(gguf.GGUFType.MODEL)

@@ -1507,6 +1597,9 @@ class TextModel(ModelBase):
        if chkhsh == "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6":
            # ref: https://huggingface.co/tencent/Hunyuan-4B-Instruct
            res = "hunyuan-dense"
+        if chkhsh == "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c":
+            # ref: https://huggingface.co/tencent/Hy4-preview
+            res = "hy_v4"
        if chkhsh == "a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6":
            # ref: https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base
            res = "falcon-h1"
@@ -1540,6 +1633,9 @@ class TextModel(ModelBase):
        if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":
            # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B
            res = "lfm2"
+        if chkhsh == "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed":
+            # ref: https://huggingface.co/XHToken/Spark-X2.5-1.7B
+            res = "spark2_5"
        if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
            # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
            res = "llama-bpe"
@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import re
+from typing import Iterable
+
+import torch
+
+from .base import ModelBase, gguf, logger
+from .deepseek import DeepseekV2Model
+
+
+def split_gate_up(weight: torch.Tensor, moe_intermediate_size: int):
+    """Split a fused stacked gate_up expert tensor into (gate, up).
+
+    weight: [n_expert, 2*moe_intermediate_size, hidden] (gate first, up second).
+    Returns (gate, up) each [n_expert, moe_intermediate_size, hidden].
+    """
+    assert weight.shape[1] == 2 * moe_intermediate_size, f"{weight.shape[1]} != 2*{moe_intermediate_size}"
+    gate = weight[:, :moe_intermediate_size, :].contiguous()
+    up = weight[:, moe_intermediate_size:, :].contiguous()
+    return gate, up
+
+
+@ModelBase.register("HYV4ForCausalLM")
+@ModelBase.example("tencent/Hy4-preview")
+class HYV4Model(DeepseekV2Model):
+    """HY_V4: DeepSeek-V3 style MLA + MoE with iHC, a gated MLA output and a learnable sink.
+
+    Reuses DeepseekV2Model for the vocab and the MLA metadata, but overrides the tensor mapping
+    because HY_V4 ships pre-stacked / fused experts plus extra iHC, gate and sink tensors. The
+    rope rows are mapped straight through (no permute) - the graph rotates consecutive pairs.
+
+    DSA is supported: indexer weights are exported for the layers marked "full" in indexer_types.
+    "shared" layers reuse the top-k of the last preceding full layer at inference time, so they
+    carry no indexer weights.
+
+    MTP (num_nextn_predict_layers) is dropped, so the GGUF cannot be used for speculative
+    decoding. The reference only runs the MTP layers while training or while speculating, so they
+    cannot change single-token logits.
+    """
+
+    model_arch = gguf.MODEL_ARCH.HY_V4
+
+    merge_expert = False
+
+    # tensors a "full" indexer layer must carry
+    INDEXER_SUFFIXES = frozenset({
+        "self_attn.indexer.wq_b.weight",
+        "self_attn.indexer.wk.weight",
+        "self_attn.indexer.k_norm.weight",
+        "self_attn.indexer.k_norm.bias",
+        "self_attn.indexer.weights_proj.weight",
+    })
+
+    @classmethod
+    def filter_tensors(cls, item):
+        # drop MTP here, not in modify_tensors, so the weights are never read
+        if item[0].startswith("model.mtp_layers."):
+            return None
+        return super().filter_tensors(item)
+
+    def _check_indexer_hparams(self):
+        for key in ("index_n_heads", "index_head_dim", "index_topk"):
+            if key not in self.hparams:
+                raise ValueError(f"HY_V4 has DSA layers but no {key}")
+
+    def indexer_is_full(self) -> list[bool] | None:
+        """Per-layer indexer ownership, or None when the checkpoint has no DSA.
+
+        indexer_types entries are "full" (owns an indexer) or "shared" (reuses the preceding
+        full layer's top-k). Missing indexer_types with sparse layers means every sparse layer
+        owns one.
+        """
+        hparams = self.hparams
+        n_layer = hparams["num_hidden_layers"]
+        indexer_types = hparams.get("indexer_types")
+
+        # the reference drives DSA off indexer_types alone; layer_types is only a fallback for
+        # checkpoints predating it (it was renamed to deepseek_sparse_attention upstream)
+        if indexer_types is None:
+            layer_types = hparams.get("layer_types") or []
+            sparse = {"sparse_attention", "deepseek_sparse_attention"}
+            if not any(t in sparse for t in layer_types):
+                return None
+            if len(layer_types) < n_layer:
+                raise ValueError(f"HY_V4 layer_types has {len(layer_types)} entries, need {n_layer}")
+            self._check_indexer_hparams()
+            return [t in sparse for t in layer_types[:n_layer]]
+
+        self._check_indexer_hparams()
+
+        if len(indexer_types) < n_layer:
+            raise ValueError(f"HY_V4 indexer_types has {len(indexer_types)} entries, need {n_layer}")
+        unknown = {t for t in indexer_types[:n_layer]} - {"full", "shared"}
+        if unknown:
+            raise ValueError(f"HY_V4 unknown indexer_types values: {sorted(unknown)}")
+        is_full = [t == "full" for t in indexer_types[:n_layer]]
+        if is_full and not is_full[0]:
+            raise ValueError("HY_V4 layer 0 must be indexer_types 'full' (nothing precedes it to share)")
+        return is_full
+
+    def set_gguf_parameters(self):
+        hparams = self.hparams
+
+        # HY4 has n_group == topk_group == 1 (no group routing). Drop the keys so the base does
+        # not emit expert_group_count/used; llama.cpp then takes the ungrouped MoE path.
+        if hparams.get("n_group") == 1 and hparams.get("topk_group") == 1:
+            hparams.pop("n_group", None)
+            hparams.pop("topk_group", None)
+
+        # HY_V4 config expresses dense/sparse layers via mlp_layer_types, but DeepseekV2Model
+        # needs first_k_dense_replace. Derive it as the contiguous leading "dense" block
+        # (the real config.json also carries first_k_dense_replace; prefer it when present,
+        # but assert the two agree so a mismatch fails loudly).
+        mlp_types = hparams.get("mlp_layer_types")
+        explicit = hparams.get("first_k_dense_replace")
+        derived = None
+        if mlp_types is not None:
+            lead = 0
+            for t in mlp_types:
+                if t == "dense":
+                    lead += 1
+                else:
+                    break
+            if any(t == "dense" for t in mlp_types[lead:]):
+                raise NotImplementedError("HY_V4 converter expects a contiguous leading dense block")
+            derived = lead
+        if explicit is not None and derived is not None and explicit != derived:
+            raise ValueError(
+                f"HY_V4 first_k_dense_replace ({explicit}) disagrees with mlp_layer_types "
+                f"leading-dense count ({derived})"
+            )
+        if explicit is None:
+            if derived is None:
+                raise ValueError("HY_V4 needs first_k_dense_replace or mlp_layer_types to place dense layers")
+            hparams["first_k_dense_replace"] = derived
+
+        # reuse DeepseekV2 MLA + MoE metadata (forces num_key_value_heads=1, writes q/kv lora,
+        # key/value lengths, expert counts, weights scale/norm, rope dims, etc.)
+        super().set_gguf_parameters()
+
+        # HY4 uses DeepSeek-V3 sigmoid routing with e_score_correction_bias. The config has no
+        # scoring_func key, so the base does not write a gating func; set it explicitly.
+        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+
+        # routed-expert SwiGLU logits clamp (only routed experts; shared/dense are not clamped,
+        # so swiglu_clamp_shexp is intentionally not written). 0.0 disables the clamp.
+        swiglu_limit = float(hparams.get("swiglu_limit", 0.0) or 0.0)
+        if swiglu_limit > 0.0:
+            self.gguf_writer.add_swiglu_clamp_exp([swiglu_limit] * self.block_count)
+
+        # iHC (independent Hyper-Connections)
+        self.gguf_writer.add_hyper_connection_count(hparams["hc_mult"])
+        self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"])
+        self.gguf_writer.add_hyper_connection_magnitude(hparams["hc_magnitude"])
+
+        # is_full is written explicitly; the graph must not infer it from tensor presence
+        is_full = self.indexer_is_full()
+        if is_full is not None:
+            self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"])
+            self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"])
+            self.gguf_writer.add_indexer_top_k(hparams["index_topk"])
+            self.gguf_writer.add_indexer_types(is_full)
+            logger.info(
+                "HY_V4 DSA: %d/%d layers own an indexer (top_k=%d, n_heads=%d, head_dim=%d)",
+                sum(is_full), len(is_full), hparams["index_topk"],
+                hparams["index_n_heads"], hparams["index_head_dim"],
+            )
+
+        if hparams.get("num_nextn_predict_layers", 0):
+            logger.warning(
+                "HY_V4: dropping %d MTP (nextn) layer(s) - the reference runs them only under "
+                "training / speculative decoding. This GGUF cannot be used for speculative decoding.",
+                hparams["num_nextn_predict_layers"],
+            )
+
+    def prepare_tensors(self):
+        # Hy4-preview for some reason has num_key_value_heads equal to 8, so override it here
+        # without this conversion/deepseek.py fails on assert
+        self.hparams["num_key_value_heads"] = self.hparams["num_attention_heads"]
+
+        # validate before the base materializes tensors, so a mismatch fails early
+        is_full = self.indexer_is_full()
+        if is_full is not None:
+            present: dict[int, set[str]] = {}
+            for name in self.model_tensors:
+                m = re.match(r"model\.layers\.(\d+)\.(self_attn\.indexer\..+)$", name)
+                if m:
+                    present.setdefault(int(m.group(1)), set()).add(m.group(2))
+            for il, expect_full in enumerate(is_full):
+                seen = present.get(il, set())
+                if expect_full and seen != self.INDEXER_SUFFIXES:
+                    raise ValueError(
+                        f"HY_V4 layer {il} is indexer_types 'full' but is missing indexer tensors: "
+                        f"{sorted(self.INDEXER_SUFFIXES - seen)}"
+                    )
+                if not expect_full and seen:
+                    raise ValueError(
+                        f"HY_V4 layer {il} is indexer_types 'shared' but carries indexer tensors: "
+                        f"{sorted(seen)}"
+                    )
+
+        super().prepare_tensors()
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        # iHC mixing matrices are 2D .weight tensors that the reference keeps in fp32
+        # (_keep_in_fp32_modules_strict). 1D tensors (hc_base/scale, attn_sinks,
+        # e_score_correction_bias) and the router (FFN_GATE_INP) are already forced F32 by the
+        # base rules. Force the HC *_fn matrices here.
+        if new_name.endswith(("hc_attn_fn.weight", "hc_ffn_fn.weight", "output_hc_fn.weight")):
+            return gguf.GGMLQuantizationType.F32
+        # indexer k_norm is fp32 in the reference; the base rules already cover
+        # *_norm.weight and INDEXER_PROJ, but not this bias
+        if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.INDEXER_K_NORM, bid, suffix=".bias"):
+            return gguf.GGMLQuantizationType.F32
+        # enable_lm_head_fp32: mirror the reference fp32 LM-head matmul by keeping output F32.
+        if new_name == "output.weight" and self.hparams.get("enable_lm_head_fp32", False):
+            return gguf.GGMLQuantizationType.F32
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def modify_tensors(self, data_torch: torch.Tensor, name: str, bid: int | None) -> Iterable[tuple[str, torch.Tensor]]:
+        hparams = self.hparams
+        moe_inter = hparams["moe_intermediate_size"]
+
+        tn = self.format_tensor_name
+
+        # fused stacked experts: split gate_up into gate/up
+        if name.endswith("mlp.experts.gate_up_proj"):
+            gate, up = split_gate_up(data_torch, moe_inter)
+            yield from super().modify_tensors(gate, tn(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), bid)
+            yield from super().modify_tensors(up,   tn(gguf.MODEL_TENSOR.FFN_UP_EXP,   bid), bid)
+            return
+
+        # add .weight suffixes
+        if name.endswith("mlp.experts.down_proj") or name.endswith(".self_attn.learnable_sink_param"):
+            name += ".weight"
+
+        if re.search(r"\.hc_head\.hc_head_(?:fn|base|scale)$", name):
+            name += ".weight"
+
+        if re.search(r"\.hc_(?:attn|mlp)_layer\.hc_pre\.hc_(?:fn|base|scale)$", name):
+            name += ".weight"
+
+        yield from super().modify_tensors(data_torch, name, bid)
@@ -379,6 +379,13 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
        self.gguf_writer.add_ssm_group_count(self.hparams["linear_num_key_heads"])
        self.gguf_writer.add_ssm_time_step_rank(self.hparams["linear_num_value_heads"])
        self.gguf_writer.add_ssm_inner_size(self.hparams["linear_value_head_dim"] * self.hparams["linear_num_value_heads"])
+        if (layer_types := self.hparams.get("layer_types")) is not None:
+            n_layer = self.hparams["num_hidden_layers"]
+            if len(layer_types) != n_layer:
+                raise ValueError(f"layer_types has {len(layer_types)} entries, expected num_hidden_layers ({n_layer})")
+            recurrent = [t == "linear_attention" for t in layer_types]
+            recurrent += [False] * (self.block_count - n_layer)
+            self.gguf_writer.add_recurrent_layers(recurrent)
        self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4))
        if (rope_dim := self.hparams.get("head_dim")) is None:
            rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf
+
+
+@ModelBase.register("Spark2_5ForCausalLM")
+@ModelBase.example("XHToken/Spark-X2.5-1.7B")
+class Spark2_5Model(TextModel):
+    model_arch = gguf.MODEL_ARCH.SPARK2_5
+
+    def set_gguf_parameters(self) -> None:
+        super().set_gguf_parameters()
+
+        hparams = self.hparams
+        layer_types = hparams["layer_types"]
+        if len(layer_types) != self.block_count:
+            raise ValueError(
+                f"Spark2_5 layer_types length {len(layer_types)} != num_hidden_layers {self.block_count}"
+            )
+        if any(layer_type not in ("sliding_attention", "full_attention") for layer_type in layer_types):
+            raise ValueError(f"Spark2_5 has unsupported layer_types: {layer_types}")
+        if hparams.get("gate_attn_act_mode") != "sigmoid" or hparams.get("headwise_attn_output_gate") is not True:
+            raise ValueError("Spark2_5 conversion requires head-wise sigmoid attention gates")
+        if hparams.get("hidden_act") != "gelu":
+            raise ValueError(f"Spark2_5 conversion requires GELU, got {hparams.get('hidden_act')!r}")
+
+        self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+        self.gguf_writer.add_sliding_window(hparams["sliding_window"])
+        self.gguf_writer.add_sliding_window_pattern(
+            [layer_type == "sliding_attention" for layer_type in layer_types]
+        )
+
+        head_dim = hparams["head_dim"]
+        full_rope = self.rope_parameters["full_attention"]
+        swa_rope = self.rope_parameters["sliding_attention"]
+        self.gguf_writer.add_rope_dimension_count(
+            int(head_dim * float(full_rope["partial_rotary_factor"]))
+        )
+        self.gguf_writer.add_rope_dimension_count_swa(
+            int(head_dim * float(swa_rope["partial_rotary_factor"]))
+        )
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name.endswith(".self_attn.q_k_v_proj.weight"):
+            if bid is None:
+                raise ValueError(f"Spark2_5 fused QKV tensor has no block id: {name}")
+            yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid), data_torch
+            return
+
+        if name.endswith(".self_attn.g_proj.weight"):
+            if bid is None:
+                raise ValueError(f"Spark2_5 attention gate tensor has no block id: {name}")
+            expected = self.hparams["num_attention_heads"]
+            if data_torch.shape[0] != expected:
+                raise ValueError(
+                    f"Spark2_5 layer {bid} attention gate width {data_torch.shape[0]} != head count {expected}"
+                )
+
+        yield from super().modify_tensors(data_torch, name, bid)
@@ -157,6 +157,10 @@ def parse_args() -> argparse.Namespace:
        help="Store tensors dequantized from FP8 as Q8_0 instead of BF16/F16.",
    )

+    parser.add_argument(
+        "--fuse-qkv", action="store_true",
+        help="Fuse separate Q, K, V weight tensors into a single QKV tensor.",
+    )
    parser.add_argument(
        "--target-model-dir", type=str, default=None,
        help=(
@@ -290,6 +294,7 @@ def main() -> None:
                                     target_model_dir=Path(args.target_model_dir) if args.target_model_dir else None,
                                     fuse_gate_up_exps=args.fuse_gate_up_exps,
                                     fp8_as_q8=args.fp8_as_q8,
+                                     fuse_qkv=args.fuse_qkv,
                                     )

        if args.vocab_only:
@@ -176,6 +176,7 @@ pre_computed_hashes = [
    {"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", "chkhsh": "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35"},
    {"name": "hunyuan", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-A13B-Instruct", "chkhsh": "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664"},
    {"name": "hunyuan-dense", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-4B-Instruct", "chkhsh": "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6"},
+    {"name": "hy_v4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hy4-preview", "chkhsh": "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c"},
    # falcon-h1 series uses 4 different tokenizers across model sizes (0.5b - 34b), hence we need to define 4 different hashes
    {"name": "falcon-h1", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base", "chkhsh": "a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6"},
    {"name": "falcon-h1", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon-H1-1B-Base", "chkhsh": "60476e1243776c4fb1b993dbd7a5f15ac22f83c80afdf425fa5ae01c8d44ef86"},
@@ -190,6 +191,7 @@ pre_computed_hashes = [
    {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
    # lfm2 variants
    {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},
+    {"name": "spark2_5", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/XHToken/Spark-X2.5-1.7B", "chkhsh": "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed"},
 ]


@@ -514,6 +514,7 @@ The following templates have active tests in `tests/test-chat.cpp`:
 | Mistral Small 3.2 | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` with call ID |
 | Devstral | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` without call ID |
 | StepFun 3.5 Flash | TAG_WITH_TAGGED | `<function=X><parameter=Y>` format |
+| Spark2.5 | TAG_WITH_TAGGED | `<tool_call>name<arg_key>...<arg_value>...` format |

 ## Adding Support for New Templates

@@ -805,8 +805,10 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. |
 | GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` |
 | GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. |
+| GGML_SYCL_MEMTRACE | 0 (default), 1, 2 | Enable record and output memory allocation diagnostics. Requires `-lv 4`. <br>0 -  Disable<br>1 - Basic memory info, including current and peak allocations, as well allocations from other sources, around 50 lines per model load.<br>2 - More verbose, logging around 900 specific allocations and deallocations. |
+| GGML_SYCL_MEMTRACE_STEP | 64 (default) or positive integer | With GGML_SYCL_MEMTRACE=1, the minimum growth in memory usage to trigger another log record. |
 | GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. |
-| GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. |
+| GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. Unsupported types and layouts fall back to the standalone op kernels. See `ggml_sycl_can_fuse()`. |
 | GGML_SYCL_ENABLE_ESIMD | 0 or 1 (default)| Enable ESIMD kernels when available. |
 | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.<br>Recommended to use when --split-mode = layer |
 | UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. |
@@ -27,6 +27,7 @@ The following sections describe how to build with different backends and options
 * [OpenCL](#opencl)
 * [Android](#android-1)
 * [OpenVINO](#openvino)
+* [Hexagon](#hexagon)
 * [Notes about GPU-accelerated backends](#notes-about-gpu-accelerated-backends)

 ## CPU Build
@@ -299,7 +300,6 @@ The following compilation options are also available to tweak performance:
 |-------------------------------|------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | GGML_CUDA_FORCE_MMQ           | Boolean                | false   | Force the use of custom matrix multiplication kernels for quantized models instead of FP16 cuBLAS even if there is no int8 tensor core implementation available (affects V100, CDNA and RDNA3+). MMQ kernels are enabled by default on GPUs with int8 tensor core support. With MMQ force enabled, speed for large batch sizes will be worse but VRAM consumption will be lower. |
 | GGML_CUDA_FORCE_CUBLAS        | Boolean                | false   | Force the use of FP16 cuBLAS instead of custom matrix multiplication kernels for quantized models. There may be issues with numerical overflows (except for V100, CDNA and RDNA4 which use FP32 compute type by default) and memory use will be higher. Prompt processing may become faster on recent datacenter GPUs (the custom kernels were tuned primarily for RTX 3000/4000).   |
-| GGML_CUDA_PEER_MAX_BATCH_SIZE | Positive integer       | 128     | Maximum batch size for which to enable peer access between multiple GPUs. Peer access requires either Linux or NVLink. When using NVLink enabling peer access for larger batch sizes is potentially beneficial.                                                                                                                                                                  |
 | GGML_CUDA_FA_ALL_QUANTS       | Boolean                | false   | Compile support for all KV cache quantization type (combinations) for the FlashAttention CUDA kernels. More fine-grained control over KV cache size but compilation takes much longer.                                                                                                                                                                                           |

 ## MUSA
@@ -830,6 +830,9 @@ To read documentation for how to build on IBM Z & LinuxONE, [click here](./build

 For build instructions and usage examples, refer to [OPENVINO.md](backend/OPENVINO.md).

+### Hexagon
+
+Check [README.md](./backend/snapdragon/README.md) for target specific build and run info.

 ---
 ## Notes about GPU-accelerated backends
@@ -12,116 +12,117 @@ Legend:
 - 🟡 Partially supported by this backend
 - ❌ Not supported by this backend

-| Operation | BLAS | CANN | CPU | CUDA | ET | MTL | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
-|-----------|------|------|------|------|------|------|------|------|------|------|------|------|
-|                              ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             ADD1 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-|                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                          ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                           CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                             CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                          CONV_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
-|                       CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                          CONV_3D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              COS | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                      COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
-|               CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|          CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                           CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                    DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                     DSV4_HC_COMB | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                     DSV4_HC_POST | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                      DSV4_HC_PRE | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                              DUP | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                   FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
-|                            FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                      GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                       GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
-|                    GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ |
-|                       GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                          L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                       LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                LIGHTNING_INDEXER | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                              LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                          MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
-|                 MUL_MAT_HADAMARD | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ❌ |
-|                              NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                   OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                     OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                         OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | 🟡 |
-|                              PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
-|                   PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                          POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                          POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                      REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                         RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                    RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                             ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                             ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                        RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              SET | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
-|                         SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ | ❌ |
-|                              SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                          SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
-|                              SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                    SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
-|                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
-|                              SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
-|                             STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
-|                         SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
-|                           SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                       SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|               TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
-|                              TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                          UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            XIELU | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+| Operation | BLAS | CANN | CPU | CUDA | ET | HTP | MTL | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
+|-----------|------|------|------|------|------|------|------|------|------|------|------|------|------|
+|                              ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             ADD1 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                          ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                           CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                             CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                          CONV_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
+|                       CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                          CONV_3D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                              COS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                      COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                              CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
+|               CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|          CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                           CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                    DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                              DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                     DSV4_HC_COMB | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                     DSV4_HC_POST | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                      DSV4_HC_PRE | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                              DUP | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                              ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                   FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
+|                            FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                            GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                      GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                       GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
+|                    GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ |
+|                       GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                          L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                       LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                LIGHTNING_INDEXER | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                              LOG | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                          MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
+|                 MUL_MAT_HADAMARD | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ❌ |
+|                              NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                   OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                     OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                         OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | 🟡 |
+|                              PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                   PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                          POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                          POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                            REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                      REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                         RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                    RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                             ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                             ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                            ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                        RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                            SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                         SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ | ❌ |
+|                              SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                          SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
+|                              SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                    SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
+|                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                              SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
+|                             STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
+|                         SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                           SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                     SWIGLU_CLAMP | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                       SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|               TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                            TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
+|                              TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                          UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            XIELU | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
@@ -734,6 +734,9 @@ class SchemaConverter:
            )
            optional_props.append("*")

+        if not required_props and not optional_props:
+            return '"{" space "}"'
+
        rule = '"{" space '
        rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props)

@@ -2,8 +2,9 @@
 #include <cstdio>

 int main(void) {
-    printf("[test-cmake] version: %s, build: %d (%s)\n",
+    printf("[test-cmake] llama.cpp version: %s, build: %d (%s)\n",
           llama_version(), LLAMA_BUILD_NUMBER, LLAMA_BUILD_COMMIT);
+    printf("[test-cmake] ggml version: %s, commit: %s\n", ggml_version(), ggml_commit());
    printf("[test-cmake] Initializing backend...\n");
    llama_backend_init();
    printf("[test-cmake] Backend initialized.\n");
@@ -128,7 +128,7 @@
          }:
          {
            # For standardised reproducible formatting with `nix fmt`
-            formatter = pkgs.nixfmt-rfc-style;
+            formatter = pkgs.nixfmt;

            # Unlike `.#packages`, legacyPackages may contain values of
            # arbitrary types (including nested attrsets) and may even throw
@@ -156,7 +156,7 @@
                windows = config.legacyPackages.llamaPackagesWindows.llama-cpp;
                python-scripts = config.legacyPackages.llamaPackages.python-scripts;
              }
-              // lib.optionalAttrs pkgs.stdenv.isLinux {
+              // lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
                cuda = config.legacyPackages.llamaPackagesCuda.llama-cpp;

                mpi-cpu = config.packages.default.override { useMpi = true; };
@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)

 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 22)
+set(GGML_VERSION_MINOR 23)
 set(GGML_VERSION_PATCH 0)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")

@@ -200,8 +200,6 @@ option(GGML_CUDA                            "ggml: use CUDA"
 option(GGML_MUSA                            "ggml: use MUSA"                                  OFF)
 option(GGML_CUDA_FORCE_MMQ                  "ggml: use mmq kernels instead of cuBLAS"         OFF)
 option(GGML_CUDA_FORCE_CUBLAS               "ggml: always use cuBLAS instead of mmq kernels"  OFF)
-set   (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
-                                            "ggml: max. batch size for using peer access")
 option(GGML_CUDA_NO_PEER_COPY               "ggml: do not use peer to peer copies"            OFF)
 option(GGML_CUDA_NO_VMM                     "ggml: do not try to use CUDA VMM"                OFF)
 option(GGML_CUDA_FA                         "ggml: compile ggml FlashAttention CUDA kernels"  ON)
@@ -406,10 +404,6 @@ write_basic_package_version_file(
    VERSION ${GGML_INSTALL_VERSION}
    COMPATIBILITY SameMajorVersion)

-target_compile_definitions(ggml-base PRIVATE
-    GGML_VERSION="${GGML_INSTALL_VERSION}"
-    GGML_COMMIT="${GGML_BUILD_COMMIT}"
-)
 message(STATUS "ggml version: ${GGML_INSTALL_VERSION}")
 message(STATUS "ggml commit:  ${GGML_BUILD_COMMIT}")

@@ -424,10 +424,6 @@ extern "C" {
    // Compare the output of two backends
    GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes);

-    // returns true for ops that may require additional memory for fleeting data on some backends,
-    // i.e. the backend's get_alloc_size may return more than ggml_nbytes for the output tensor
-    GGML_API bool ggml_backend_op_alloc_size_may_expand(enum ggml_op op);
-
    // Tensor initialization
    GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
    GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor);
@@ -213,7 +213,9 @@ set_target_properties(ggml-base PROPERTIES
    SOVERSION ${GGML_VERSION_MAJOR}
 )

-target_include_directories(ggml-base PRIVATE .)
+configure_file(ggml-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.h @ONLY)
+
+target_include_directories(ggml-base PRIVATE . ${CMAKE_CURRENT_BINARY_DIR})
 if (GGML_BACKEND_DL)
    target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL)
 endif()
@@ -34,6 +34,11 @@ extern "C" {
        void * context;
    };

+    // [TAG_ALLOC_SIZE_EXPAND]
+    // returns true for ops that may require additional memory for fleeting data on some backends,
+    // i.e. the backend buffer type's get_alloc_size may return more than ggml_nbytes for the output tensor
+    GGML_API bool ggml_op_alloc_size_may_expand(enum ggml_op op);
+
    //
    // Backend buffer
    //
@@ -490,7 +490,13 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
 #endif
        // default search paths: executable directory, current directory
        search_paths.push_back(get_executable_path());
-        search_paths.push_back(fs::current_path());
+        std::error_code cwd_ec;
+        const fs::path cwd = fs::current_path(cwd_ec);
+        if (cwd_ec) {
+            GGML_LOG_DEBUG("%s: current_path() failure, error-message: %s\n", __func__, cwd_ec.message().c_str());
+        } else {
+            search_paths.push_back(cwd);
+        }
    } else {
        search_paths.push_back(fs::u8path(user_search_path));
    }
@@ -508,8 +514,14 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
            }
            continue;
        }
-        fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);
-        for (const auto & entry : dir_it) {
+        std::error_code dir_ec;
+        fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied, dir_ec);
+        if (dir_ec) {
+            GGML_LOG_DEBUG("%s: failed to enumerate %s: %s\n", __func__, path_str(search_path).c_str(), dir_ec.message().c_str());
+            continue;
+        }
+        for (const fs::directory_iterator end; dir_it != end; dir_it.increment(dir_ec)) {
+            const auto & entry = *dir_it;
            if (entry.is_regular_file(ec)) {
                auto filename = entry.path().filename();
                auto ext = entry.path().extension();
@@ -71,7 +71,7 @@ size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const s
        GGML_ASSERT(size <= ggml_nbytes(tensor) ||
                    ggml_op_is_empty(tensor->op) ||
                    ggml_is_quantized(tensor->type) || // [TAG_ALLOC_SIZE_EXPAND]
-                    ggml_backend_op_alloc_size_may_expand(tensor->op));
+                    ggml_op_alloc_size_may_expand(tensor->op));

        return size;
    }
@@ -849,7 +849,7 @@ static void ggml_backend_sched_split_inputs_grow(struct ggml_backend_sched_split
    int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;
    if (split->inputs_capacity > 0) {
        new_cap = 2*split->inputs_capacity;
-        GGML_LOG_WARN("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap);
+        GGML_LOG_DEBUG("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap);
    }
    auto * pnew = (struct ggml_tensor **) realloc((void *) split->inputs, new_cap * sizeof(struct ggml_tensor *));
    if (pnew == NULL) {
@@ -864,7 +864,7 @@ static void ggml_backend_sched_graph_inputs_grow(ggml_backend_sched_t sched) {
    int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;
    if (sched->graph_inputs_capacity > 0) {
        new_cap = 2*sched->graph_inputs_capacity;
-        GGML_LOG_WARN("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap);
+        GGML_LOG_DEBUG("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap);
    }
    auto * pnew = (struct ggml_tensor **) realloc((void *) sched->graph_inputs, new_cap * sizeof(struct ggml_tensor *));
    if (pnew == NULL) {
@@ -1338,17 +1338,6 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
                            break;
                        }
                    }
-                    // check if the split has too many inputs
-                    // FIXME: count the number of inputs instead of only checking when full
-                    if (split->n_inputs >= split->inputs_capacity) {
-                        const size_t id = hash_id(src);
-                        int src_backend_id = sched->hv_tensor_backend_ids[id];
-                        bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);
-                        if (src_backend_id != cur_backend_id && tensor_id_copy(id, cur_backend_id, 0) == NULL && !supported) {
-                            need_new_split = true;
-                            break;
-                        }
-                    }
                }
            }

@@ -2109,10 +2098,7 @@ ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched,

 // utils

-// [TAG_ALLOC_SIZE_EXPAND]
-// returns true for ops that may require additional memory for fleeting data on some backends,
-// i.e. the backend's get_alloc_size may return more than ggml_nbytes for the output tensor
-bool ggml_backend_op_alloc_size_may_expand(enum ggml_op op) {
+bool ggml_op_alloc_size_may_expand(enum ggml_op op) {
    switch (op) {
        case GGML_OP_FLASH_ATTN_EXT:
        case GGML_OP_MUL_MAT:
@@ -636,7 +636,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
        const float32x4_t v_xyf = vec_float(v_xy);

        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));
-        const float32x4_t v_acc = vec_madd(v_xyf, v_d, v_acc);
+        const float32x4_t v_acc = vec_madd(v_xyf, v_d, vec_splats(0.0f));

        sumf += vec_hsum_f32x4(v_acc) + summs;
    }
@@ -129,8 +129,6 @@ if (CUDAToolkit_FOUND)
                             ${GGML_SOURCES_CUDA}
                            )

-    add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})
-
    if (GGML_CUDA_GRAPHS)
        add_compile_definitions(GGML_CUDA_USE_GRAPHS)
    endif()
@@ -52,6 +52,7 @@
 #define GGML_CUDA_CC_VOLTA           700
 #define GGML_CUDA_CC_TURING          750
 #define GGML_CUDA_CC_AMPERE          800
+#define GGML_CUDA_CC_ORIN            870
 #define GGML_CUDA_CC_ADA_LOVELACE    890
 #define GGML_CUDA_CC_HOPPER          900
 // While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see
@@ -68,6 +69,8 @@
 #define GGML_CUDA_CC_GCN4       (GGML_CUDA_CC_OFFSET_AMD + 0x803)  // Tonga, Fiji, Polaris, minimum for fast fp16
 #define GGML_CUDA_CC_VEGA       (GGML_CUDA_CC_OFFSET_AMD + 0x900)  // Vega56/64, minimum for fp16 dual issue
 #define GGML_CUDA_CC_VEGA20     (GGML_CUDA_CC_OFFSET_AMD + 0x906)  // MI50/Radeon VII, minimum for dp4a
+#define GGML_CUDA_CC_GFX909     (GGML_CUDA_CC_OFFSET_AMD + 0x909)  // GCN APU
+#define GGML_CUDA_CC_GFX90C     (GGML_CUDA_CC_OFFSET_AMD + 0x90c)  // GCN APU
 #define GGML_CUDA_CC_CDNA1      (GGML_CUDA_CC_OFFSET_AMD + 0x908)  // MI100, minimum for MFMA, acc registers
 #define GGML_CUDA_CC_CDNA2      (GGML_CUDA_CC_OFFSET_AMD + 0x90a)  // MI210 (gfx90a), minimum acc register renaming
 #define GGML_CUDA_CC_CDNA3      (GGML_CUDA_CC_OFFSET_AMD + 0x942)  // MI300
@@ -88,12 +91,13 @@
 #define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4)
 #define GGML_CUDA_CC_IS_RDNA3(cc)   (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc))
 #define GGML_CUDA_CC_IS_RDNA4(cc)   (cc >= GGML_CUDA_CC_RDNA4)
-#define GGML_CUDA_CC_IS_GCN(cc)     (cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1)
-#define GGML_CUDA_CC_IS_CDNA(cc)    (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1)
-#define GGML_CUDA_CC_IS_CDNA1(cc)   (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2)
-#define GGML_CUDA_CC_IS_CDNA2(cc)   (cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3)
-#define GGML_CUDA_CC_IS_CDNA3(cc)   (cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4)
-#define GGML_CUDA_CC_IS_CDNA4(cc)   (cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1)
+#define GGML_CUDA_CC_IS_GCN_APU(cc) ((cc) == GGML_CUDA_CC_GFX909 || (cc) == GGML_CUDA_CC_GFX90C)
+#define GGML_CUDA_CC_IS_GCN(cc)     ((cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) || GGML_CUDA_CC_IS_GCN_APU(cc))
+#define GGML_CUDA_CC_IS_CDNA(cc)    (!GGML_CUDA_CC_IS_GCN_APU(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1)
+#define GGML_CUDA_CC_IS_CDNA1(cc)   (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2)
+#define GGML_CUDA_CC_IS_CDNA2(cc)   (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3)
+#define GGML_CUDA_CC_IS_CDNA3(cc)   (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4)
+#define GGML_CUDA_CC_IS_CDNA4(cc)   (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1)

 // Moore Threads
 #define MUSART_HMASK 40300 // MUSA rc4.3, min. ver. for half2 -> uint mask comparisons
@@ -120,6 +124,12 @@
 #    define GGML_CUDA_USE_PDL
 #endif  // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080))

+static __device__ __forceinline__ void ggml_cuda_syncwarp() {
+#ifndef GGML_USE_HIP
+    __syncwarp();
+#endif // GGML_USE_HIP
+}
+
 static __device__ __forceinline__ void ggml_cuda_pdl_sync() {
 #if defined(GGML_CUDA_USE_PDL) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER
    cudaGridDependencySynchronize();
@@ -969,6 +979,7 @@ template<>
 struct ggml_cuda_type_traits<GGML_TYPE_F16> {
    static constexpr int qk = 1;
    static constexpr int qr = 1;
+    static constexpr int bs = sizeof(ggml_half);
 };

 template<>
@@ -976,6 +987,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q1_0> {
    static constexpr int qk = QK1_0;
    static constexpr int qr = QR1_0;
    static constexpr int qi = QI1_0;
+    static constexpr int bs = sizeof(block_q1_0);
 };

 template<>
@@ -983,6 +995,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q2_0> {
    static constexpr int qk = QK2_0;
    static constexpr int qr = QR2_0;
    static constexpr int qi = QI2_0;
+    static constexpr int bs = sizeof(block_q2_0);
 };

 template<>
@@ -990,6 +1003,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q4_0> {
    static constexpr int qk = QK4_0;
    static constexpr int qr = QR4_0;
    static constexpr int qi = QI4_0;
+    static constexpr int bs = sizeof(block_q4_0);
 };

 template<>
@@ -997,6 +1011,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q4_1> {
    static constexpr int qk = QK4_1;
    static constexpr int qr = QR4_1;
    static constexpr int qi = QI4_1;
+    static constexpr int bs = sizeof(block_q4_1);
 };

 template<>
@@ -1004,6 +1019,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q5_0> {
    static constexpr int qk = QK5_0;
    static constexpr int qr = QR5_0;
    static constexpr int qi = QI5_0;
+    static constexpr int bs = sizeof(block_q5_0);
 };

 template<>
@@ -1011,6 +1027,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q5_1> {
    static constexpr int qk = QK5_1;
    static constexpr int qr = QR5_1;
    static constexpr int qi = QI5_1;
+    static constexpr int bs = sizeof(block_q5_1);
 };

 template<>
@@ -1018,6 +1035,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q8_0> {
    static constexpr int qk = QK8_0;
    static constexpr int qr = QR8_0;
    static constexpr int qi = QI8_0;
+    static constexpr int bs = sizeof(block_q8_0);
 };

 template<>
@@ -1025,6 +1043,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_MXFP4> {
    static constexpr int qk = QK_MXFP4;
    static constexpr int qr = QR_MXFP4;
    static constexpr int qi = QI_MXFP4;
+    static constexpr int bs = sizeof(block_mxfp4);
 };

 template<>
@@ -1032,6 +1051,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_NVFP4> {
    static constexpr int qk = QK_NVFP4;
    static constexpr int qr = QR_NVFP4;
    static constexpr int qi = QI_NVFP4;
+    static constexpr int bs = sizeof(block_nvfp4);
 };

 template<>
@@ -1039,6 +1059,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q2_K> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR2_K;
    static constexpr int qi = QI2_K;
+    static constexpr int bs = sizeof(block_q2_K);
 };

 template<>
@@ -1046,6 +1067,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q3_K> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR3_K;
    static constexpr int qi = QI3_K;
+    static constexpr int bs = sizeof(block_q3_K);
 };

 template<>
@@ -1053,6 +1075,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q4_K> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR4_K;
    static constexpr int qi = QI4_K;
+    static constexpr int bs = sizeof(block_q4_K);
 };

 template<>
@@ -1060,6 +1083,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q5_K> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR5_K;
    static constexpr int qi = QI5_K;
+    static constexpr int bs = sizeof(block_q5_K);
 };

 template<>
@@ -1067,6 +1091,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q6_K> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR6_K;
    static constexpr int qi = QI6_K;
+    static constexpr int bs = sizeof(block_q6_K);
 };

 template<>
@@ -1074,6 +1099,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XXS> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR2_XXS;
    static constexpr int qi = QI2_XXS;
+    static constexpr int bs = sizeof(block_iq2_xxs);
 };

 template<>
@@ -1081,6 +1107,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XS> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR2_XS;
    static constexpr int qi = QI2_XS;
+    static constexpr int bs = sizeof(block_iq2_xs);
 };

 template<>
@@ -1088,6 +1115,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ2_S> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR2_S;
    static constexpr int qi = QI2_S;
+    static constexpr int bs = sizeof(block_iq2_s);
 };

 template<>
@@ -1095,6 +1123,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ3_XXS> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR3_XXS;
    static constexpr int qi = QI3_XXS;
+    static constexpr int bs = sizeof(block_iq3_xxs);
 };

 template<>
@@ -1102,6 +1131,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ1_S> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR1_S;
    static constexpr int qi = QI1_S;
+    static constexpr int bs = sizeof(block_iq1_s);
 };

 template<>
@@ -1109,6 +1139,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ1_M> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR1_M;
    static constexpr int qi = QI1_M;
+    static constexpr int bs = sizeof(block_iq1_m);
 };

 template<>
@@ -1116,6 +1147,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ4_NL> {
    static constexpr int qk = QK4_NL;
    static constexpr int qr = QR4_NL;
    static constexpr int qi = QI4_NL;
+    static constexpr int bs = sizeof(block_iq4_nl);
 };

 template<>
@@ -1123,6 +1155,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ4_XS> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR4_XS;
    static constexpr int qi = QI4_XS;
+    static constexpr int bs = sizeof(block_iq4_xs);
 };

 template<>
@@ -1130,6 +1163,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ3_S> {
    static constexpr int qk = QK_K;
    static constexpr int qr = QR3_S;
    static constexpr int qi = QI3_S;
+    static constexpr int bs = sizeof(block_iq3_s);
 };

 //////////////////////
@@ -1545,77 +1545,77 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
        }
    }

-    if (np > 1 && threadIdx.y % np == 0) {
-        // Combine the meta data for parallel warps via shared memory.
-        // Warps with threadIdx.y % np != 0 must NOT return early.
-        // All threads must return simultaneously to avoid race conditions with work on the next tile.
-
+    if (np > 1) {
        constexpr int nmeta = np*cols_per_warp >= warp_size ? np*cols_per_warp/warp_size : 1;

+        float KQ_cmn;
+        float KQ_cms[nmeta];
+        float KQ_crs;
+
        const int jc_meta = threadIdx.y*cols_per_warp + (np*cols_per_warp < warp_size ? threadIdx.x % (np*cols_per_warp) : threadIdx.x);
        float2 * const meta_ptr = ((float2 *) tile_Q) + jc_meta*(tile_stride/2) + nbatch_combine/2;
-        float2 meta[nmeta];
-#pragma unroll
-        for (int imeta = 0; imeta < nmeta; ++imeta) {
-            meta[imeta] = meta_ptr[imeta * warp_size * tile_stride/2];
-        }

-        float KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps.
+        if (threadIdx.y % np == 0) {
+            // Combine the meta data for parallel warps via shared memory.
+            float2 meta[nmeta];
 #pragma unroll
-        for (int imeta = 1; imeta < nmeta; ++imeta) {
-            KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x);
-        }
-#pragma unroll
-        for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {
-            if (offset < warp_size) {
-                KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size));
+            for (int imeta = 0; imeta < nmeta; ++imeta) {
+                meta[imeta] = meta_ptr[imeta * warp_size * tile_stride/2];
            }
-        }

-        float KQ_cms[nmeta]; // KQ combine max scale per warp.
+            KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps.
 #pragma unroll
-        for (int imeta = 0; imeta < nmeta; ++imeta) {
-            KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn);
-        }
+            for (int imeta = 1; imeta < nmeta; ++imeta) {
+                KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x);
+            }
+#pragma unroll
+            for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {
+                if (offset < warp_size) {
+                    KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size));
+                }
+            }

-        float KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps.
 #pragma unroll
-        for (int imeta = 1; imeta < nmeta; ++imeta) {
-            KQ_crs += KQ_cms[imeta]*meta[imeta].y;
-        }
+            for (int imeta = 0; imeta < nmeta; ++imeta) {
+                KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn);
+            }
+
+            KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps.
 #pragma unroll
-        for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {
-            if (offset < warp_size) {
-                KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size);
+            for (int imeta = 1; imeta < nmeta; ++imeta) {
+                KQ_crs += KQ_cms[imeta]*meta[imeta].y;
+            }
+#pragma unroll
+            for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {
+                if (offset < warp_size) {
+                    KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size);
+                }
            }
        }

        __syncthreads();

-        // Write back combined meta data:
+        if (threadIdx.y % np == 0) {
+            // Write back combined meta data:
 #pragma unroll
-        for (int imeta = 0; imeta < nmeta; ++imeta) {
-            if (np*cols_per_warp >= warp_size || threadIdx.x < np*cols_per_warp) {
-                // Combined KQ max scale + rowsum.
-                meta_ptr[imeta * warp_size * tile_stride/2] = make_float2(KQ_cms[imeta], KQ_crs);
+            for (int imeta = 0; imeta < nmeta; ++imeta) {
+                if (np*cols_per_warp >= warp_size || threadIdx.x < np*cols_per_warp) {
+                    // Combined KQ max scale + rowsum.
+                    meta_ptr[imeta * warp_size * tile_stride/2] = make_float2(KQ_cms[imeta], KQ_crs);
+                }
+            }
+
+            // Combined KQ max + rowsum.
+            static_assert(cols_per_warp <= warp_size);
+            if (needs_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) {
+                float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols;
+                dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs);
+            }
+            if (is_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) {
+                float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols;
+                dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs);
            }
        }
-
-        // Combined KQ max + rowsum.
-        static_assert(cols_per_warp <= warp_size);
-        if (needs_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) {
-            float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols;
-            dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs);
-        }
-        if (is_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) {
-            float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols;
-            dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs);
-        }
-    } else if (np > 1) {
-        // Warps with threadIdx.y % np == 0 execute a __syncthreads() in the if branch.
-        // Therefore, all other warps also need to execute a __syncthreads().
-        // Otherwise the points at which warps synchronize with each other would become misaligned.
-        __syncthreads();
    }

 #pragma unroll
@@ -317,9 +317,7 @@ static __global__ void flash_attn_ext_vec(
 #endif // V_DOT2_F32_F16_AVAILABLE
        }

-#ifndef GGML_USE_HIP
-        __syncwarp();
-#endif // GGML_USE_HIP
+        ggml_cuda_syncwarp();

 #pragma unroll
        for (int k0 = 0; k0 < WARP_SIZE; k0 += V_cols_per_iter) {
@@ -212,6 +212,7 @@ static int ggml_cuda_parse_id(char devName[]) {
    }
    archNum += archMajor * 0x100;
    archNum += archMinor;
+
    return archNum;
 }
 #endif // defined(GGML_USE_HIP)
@@ -143,6 +143,7 @@ static __global__ void mul_mat_f(
            if (threadIdx.x == 0) {
                slot_map[j] = -1;
            }
+            ggml_cuda_syncwarp();

            if (col_base + j >= ncols_dst_total) {
                continue;
@@ -171,10 +172,12 @@ static __global__ void mul_mat_f(
        tile_A A[ntA][warp_size / tile_A::J];
 #pragma unroll
        for (int itA = 0; itA < ntA; ++itA) {
+            ggml_cuda_syncwarp();
 #pragma unroll
            for (int i = 0; i < tile_A::I; ++i) {
                tile_xy[i*tile_k_padded + threadIdx.x] = x[(itA*tile_A::I + i)*stride_row  + col];
            }
+            ggml_cuda_syncwarp();
 #pragma unroll
            for (int k0 = 0; k0 < warp_size; k0 += tile_A::J) {
                load_ldmatrix(A[itA][k0/tile_A::J], tile_xy + k0, tile_k_padded);
@@ -183,6 +186,7 @@ static __global__ void mul_mat_f(

 #pragma unroll
        for (int itB = 0; itB < ntB; ++itB) {
+            ggml_cuda_syncwarp();
            if constexpr (std::is_same_v<T, float>) {
 #pragma unroll
                for (int j0 = 0; j0 < tile_B::I; ++j0) {
@@ -212,6 +216,7 @@ static __global__ void mul_mat_f(
            } else {
                static_assert(std::is_same_v<T, void>, "unsupported type");
            }
+            ggml_cuda_syncwarp();
 #pragma unroll
            for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) {
                tile_B B;
@@ -229,6 +234,8 @@ static __global__ void mul_mat_f(

    if (nwarps > 1) {
        __syncthreads();
+    } else {
+        ggml_cuda_syncwarp();
    }
 #pragma unroll
    for (int itB = 0; itB < ntB; ++itB) {
@@ -245,6 +252,8 @@ static __global__ void mul_mat_f(

    if (nwarps > 1) {
        __syncthreads();
+    } else {
+        ggml_cuda_syncwarp();
    }

 #pragma unroll
@@ -382,10 +391,12 @@ static __global__ void mul_mat_f_ids(
        tile_A A[ntA][warp_size / tile_A::J];
 #pragma unroll
        for (int itA = 0; itA < ntA; ++itA) {
+            ggml_cuda_syncwarp();
 #pragma unroll
            for (int i = 0; i < tile_A::I; ++i) {
                tile_xy[i*tile_k_padded + threadIdx.x] = x[(itA*tile_A::I + i)*stride_row  + col];
            }
+            ggml_cuda_syncwarp();
 #pragma unroll
            for (int k0 = 0; k0 < warp_size; k0 += tile_A::J) {
                load_ldmatrix(A[itA][k0/tile_A::J], tile_xy + k0, tile_k_padded);
@@ -419,6 +430,7 @@ static __global__ void mul_mat_f_ids(
            int next_buf = 1;
 #pragma unroll
            for (int itB = 0; itB < ntB; ++itB) {
+                ggml_cuda_syncwarp();
 #pragma unroll
                for (int j0 = 0; j0 < tile_B::I; ++j0) {
                    tile_xy[j0*tile_k_padded + threadIdx.x] = vals_buf[curr_buf][j0];
@@ -428,6 +440,7 @@ static __global__ void mul_mat_f_ids(
                    gather_tile(itB + 1, vals_buf[next_buf]);
                }

+                ggml_cuda_syncwarp();
 #pragma unroll
                for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) {
                    tile_B B;
@@ -472,6 +485,7 @@ static __global__ void mul_mat_f_ids(
            int next_buf = 1;
 #pragma unroll
            for (int itB = 0; itB < ntB; ++itB) {
+                ggml_cuda_syncwarp();
 #pragma unroll
                for (int j0 = 0; j0 < tile_B::I; ++j0) {
                    const float2 tmp = vals_buf[curr_buf][j0];
@@ -482,6 +496,7 @@ static __global__ void mul_mat_f_ids(
                    gather_tile(itB + 1, vals_buf[next_buf]);
                }

+                ggml_cuda_syncwarp();
 #pragma unroll
                for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) {
                    tile_B B;
@@ -507,6 +522,8 @@ static __global__ void mul_mat_f_ids(

    if (nwarps > 1) {
        __syncthreads();
+    } else {
+        ggml_cuda_syncwarp();
    }
 #pragma unroll
    for (int itB = 0; itB < ntB; ++itB) {
@@ -523,6 +540,8 @@ static __global__ void mul_mat_f_ids(

    if (nwarps > 1) {
        __syncthreads();
+    } else {
+        ggml_cuda_syncwarp();
    }

 #pragma unroll
@@ -101,6 +101,7 @@ static __global__ void mm_ids_helper(
        }
    }
    nex_prev = warp_reduce_sum<warp_size>(nex_prev);
+    ggml_cuda_syncwarp();

    for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
        const mm_ids_helper_store store_it = store[itc];
@@ -375,10 +375,10 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
        return true;
    }

-    // gfx900 (Vega 10) lacks native dp4a, loses to dequant + hipBLAS
+    // gfx900 (Vega 10), gfx909, and gfx90c lack native dp4a, losing to dequant + hipBLAS
    // for dense matrices; keep MMQ only for MoE, where the
    // hipBLAS path is much slower.
-    if (cc == GGML_CUDA_CC_VEGA) {
+    if (cc == GGML_CUDA_CC_VEGA || GGML_CUDA_CC_IS_GCN_APU(cc)) {
        return n_experts > 0;
    }

@@ -6,6 +6,35 @@
 #include <cstdint>
 #include <type_traits>

+// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel
+// has little exposed latency left to hide and the extra requests cost more than they save.
+// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
+// returns true only for those quants that benefit from prefetch and false otherwise
+static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_Q4_0:
+        case GGML_TYPE_Q5_0:
+        case GGML_TYPE_Q8_0:
+        case GGML_TYPE_MXFP4:
+        case GGML_TYPE_Q3_K:
+        case GGML_TYPE_Q4_K:
+        case GGML_TYPE_Q5_K:
+        case GGML_TYPE_Q6_K:
+        case GGML_TYPE_IQ1_M:
+        case GGML_TYPE_IQ4_NL:
+        case GGML_TYPE_IQ4_XS:
+            return true;
+        default:
+            return false;
+    }
+}
+
+static __device__ __forceinline__ void mmvq_prefetch_l2(const void * p) {
+    asm volatile("prefetch.global.L2 [%0];" :: "l"(p));
+}
+#endif
+
 typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs);

 static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) {
@@ -298,9 +327,6 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
                return ne11 <= 4;
            case GGML_TYPE_Q3_K:
                return ne11 <= 6;
-            case GGML_TYPE_Q4_K:
-            case GGML_TYPE_Q5_K:
-                return ne11 <= 7;
            default:
                return ne11 <= MMVQ_MAX_BATCH_SIZE;
        }
@@ -310,8 +336,9 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
            case GGML_TYPE_Q2_K:
            case GGML_TYPE_Q3_K:
            case GGML_TYPE_Q4_K:
-            case GGML_TYPE_Q5_K:
                return ne11 <= 5;
+            case GGML_TYPE_Q5_K:
+                return ne11 <= 6;
            case GGML_TYPE_Q6_K:
                return ne11 <= 7;
            default:
@@ -326,6 +353,18 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
                return ne11 <= MMVQ_MAX_BATCH_SIZE;
        }
    }
+    if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ORIN) {
+        switch (type) { // tuned for Jetson Orin
+            case GGML_TYPE_Q2_K:
+            case GGML_TYPE_Q3_K:
+            case GGML_TYPE_Q4_K:
+            case GGML_TYPE_Q5_K:
+            case GGML_TYPE_Q6_K:
+                return ne11 <= 1;
+            default:
+                return ne11 <= MMVQ_MAX_BATCH_SIZE;
+        }
+    }
    if (GGML_CUDA_CC_IS_CDNA(cc)) {
        if (GGML_CUDA_CC_IS_CDNA1(cc)) {
            switch (type) {
@@ -663,6 +702,26 @@ static __global__ void mul_mat_vec_q(
        // x block quant index when casting the quants to int
        const int kqs = vdr * (tid % (qi/vdr));

+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
+        // start the next iterations' weight loads early
+        if constexpr (mmvq_should_prefetch(type)) {
+            constexpr int pf_dist = 2; // loop iterations, not blocks
+            const int kbx_pf = kbx + pf_dist*blocks_per_iter;
+            if (kbx_pf < blocks_per_row_x) {
+#pragma unroll
+                for (int i = 0; i < rows_per_cuda_block; ++i) {
+                    const size_t off = (size_t)(kbx_offset + i*stride_row_x + kbx_pf) * ggml_cuda_type_traits<type>::bs;
+                    mmvq_prefetch_l2((const char *) vx + off);
+                    if constexpr (has_fusion) {
+                        if (use_gate) {
+                            mmvq_prefetch_l2((const char *) vgate + off);
+                        }
+                    }
+                }
+            }
+        }
+#endif
+
 #pragma unroll
        for (int j = 0; j < ncols_dst; ++j) {
 #pragma unroll
@@ -936,16 +936,20 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
    v[0] = q4[0];
    v[1] = q4[4];

+    // branchless so nvcc can hoist this out of the ncols_dst loop
    const uint16_t * scales = (const uint16_t *)bq4_K->scales;
+    const int j  = bq8_offset/2;
+    const int jm = j & 1;
+
+    const uint32_t s0 = scales[jm + 0];
+    const uint32_t s2 = scales[jm + 2];
+    const uint32_t s4 = scales[jm + 4];
+
+    const uint32_t hi = (uint32_t) -(int32_t) (j >= 2);
+
    uint16_t aux[2];
-    const int j = bq8_offset/2;
-    if (j < 2) {
-        aux[0] = scales[j+0] & 0x3f3f;
-        aux[1] = scales[j+2] & 0x3f3f;
-    } else {
-        aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
-        aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
-    }
+    aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi));
+    aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi));
    const uint8_t * sc = (const uint8_t *)aux;
    const uint8_t * m  = sc + 2;

@@ -981,16 +985,21 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
    vh[0] = qh[0] >> bq8_offset;
    vh[1] = qh[4] >> bq8_offset;

+    // same as q4_K
    const uint16_t * scales = (const uint16_t *)bq5_K->scales;
+    const int j  = bq8_offset/2;
+    const int jm = j & 1;
+
+    const uint32_t s0 = scales[jm + 0];
+    const uint32_t s2 = scales[jm + 2];
+    const uint32_t s4 = scales[jm + 4];
+
+    const uint32_t hi = (uint32_t) -(int32_t) (j >= 2);
+
    uint16_t aux[2];
-    const int j = bq8_offset/2;
-    if (j < 2) {
-        aux[0] = scales[j+0] & 0x3f3f;
-        aux[1] = scales[j+2] & 0x3f3f;
-    } else {
-        aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
-        aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
-    }
+    aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi));
+    aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi));
+
    const uint8_t * sc = (const uint8_t *)aux;
    const uint8_t * m  = sc + 2;

@@ -176,9 +176,9 @@

 #define __CUDA_ARCH__ 1300

-#if defined(__gfx900__) || defined(__gfx906__)
+#if defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__)
 #define GCN5
-#endif // defined(__gfx900__) || defined(__gfx906__)
+#endif // defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__)

 #if defined(__gfx803__)
 #define GCN4
@@ -111,6 +111,7 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
        id<MTLCommandQueue> queue = ggml_metal_device_get_queue(dev);
        if (queue == nil) {
            GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
+            free(res);
            return NULL;
        }

@@ -1577,6 +1577,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(
    return res;
 }

+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(
+        ggml_metal_library_t lib,
+        const ggml_tensor * op) {
+    assert(op->op == GGML_OP_FLASH_ATTN_EXT);
+    assert(op->src[3]);
+
+    char name[256];
+
+    snprintf(name, 256, "kernel_flash_attn_ext_vec_idx");
+
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+    }
+
+    GGML_UNUSED(op);
+
+    return res;
+}
+
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec(
        ggml_metal_library_t lib,
        const ggml_tensor * op,
@@ -1585,6 +1605,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
        bool    has_bias,
        bool    has_scap,
        bool    has_kvpad,
+        bool    has_sparse,
        int32_t nqpsg,
        int32_t ne,
        int32_t nsg,
@@ -1614,13 +1635,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
            dv,
            qne_suffix);

-    snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d",
+    snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_sparse=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d",
            base,
            has_mask,
            has_sinks,
            has_bias,
            has_scap,
            has_kvpad,
+            has_sparse,
            ns10,
            ns20,
            nsg, nwg);
@@ -1633,7 +1655,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
        ggml_metal_cv_set_bool(cv, has_sinks, FC_FLASH_ATTN_EXT_VEC + 1);
        ggml_metal_cv_set_bool(cv, has_bias,  FC_FLASH_ATTN_EXT_VEC + 2);
        ggml_metal_cv_set_bool(cv, has_scap,  FC_FLASH_ATTN_EXT_VEC + 3);
-        ggml_metal_cv_set_bool(cv, has_kvpad, FC_FLASH_ATTN_EXT_VEC + 4);
+        ggml_metal_cv_set_bool(cv, has_kvpad,  FC_FLASH_ATTN_EXT_VEC + 4);
+        ggml_metal_cv_set_bool(cv, has_sparse, FC_FLASH_ATTN_EXT_VEC + 5);

        ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT_VEC + 20);
        ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT_VEC + 21);
@@ -201,6 +201,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att
        int32_t ns10,
        int32_t ns20);

+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(
+        ggml_metal_library_t lib,
+        const struct ggml_tensor * op);
+
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec(
        ggml_metal_library_t lib,
        const struct ggml_tensor * op,
@@ -209,6 +213,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att
        bool    has_bias,
        bool    has_scap,
        bool    has_kvpad,
+        bool    has_sparse,
        int32_t nqpsg,
        int32_t ne,
        int32_t nsg,
@@ -1486,7 +1486,9 @@ static bool ggml_metal_supports_mul_mat_op(
        const struct ggml_tensor * op,
        bool src0_f16_has_mv,
        bool mm_path) {
-    if (!has_simdgroup_reduction || op->src[0]->type == GGML_TYPE_NVFP4) {
+    if (!has_simdgroup_reduction ||
+        op->src[0]->type == GGML_TYPE_NVFP4 ||
+        op->src[0]->type == GGML_TYPE_TQ1_0) {
        return false;
    }

@@ -1887,7 +1889,8 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                };
            }
        case GGML_OP_GET_ROWS:
-            return op->src[0]->type != GGML_TYPE_NVFP4;
+            return op->src[0]->type != GGML_TYPE_NVFP4 &&
+                   op->src[0]->type != GGML_TYPE_TQ1_0;
        case GGML_OP_SET_ROWS:
            {
                if (op->src[0]->type == GGML_TYPE_F16) {
@@ -458,8 +458,21 @@ typedef struct {
    float    m1;
    int32_t  n_head_log2;
    float    logit_softcap;
+    int32_t  n_kv_max_padded;
 } ggml_metal_kargs_flash_attn_ext_vec;

+typedef struct {
+    int32_t  ne30;
+    int32_t  ne31;
+    int32_t  ne32;
+    int32_t  ne33;
+    uint64_t nb31;
+    uint64_t nb32;
+    uint64_t nb33;
+    int32_t  n_kv_max;
+    int32_t  n_kv_max_padded;
+} ggml_metal_kargs_flash_attn_ext_vec_idx;
+
 typedef struct {
    int32_t  nrows;
 } ggml_metal_kargs_flash_attn_ext_vec_reduce;
@@ -917,7 +917,7 @@ int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) {

    const int64_t nrows = ggml_nrows(op->src[0]);

-    const int32_t nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2);
+    const int32_t nth = std::max(1, std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2));

    ggml_metal_encoder_set_pipeline(enc, pipeline);
    ggml_metal_encoder_set_bytes   (enc, &args, sizeof(args), 0);
@@ -2857,6 +2857,65 @@ static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) {
    }
 }

+// returns the n_kv_max hint if the sparse path is available for this op, or 0 otherwise
+// the mask (src[3]) remains the single source of truth: finite entries are the valid KV positions,
+// n_kv_max is only an upper bound on their number per mask row, used to size the index lists
+static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) {
+    assert(op->op == GGML_OP_FLASH_ATTN_EXT);
+
+    int32_t n_kv_max = 0;
+    memcpy(&n_kv_max, ((const int32_t *) op->op_params) + 4, sizeof(n_kv_max));
+
+    if (n_kv_max <= 0) {
+        return 0;
+    }
+
+    // the sparse indices are gathered from the mask
+    if (!op->src[3]) {
+        return 0;
+    }
+
+    // bound the size of the index lists
+    if (n_kv_max > 4096) {
+        return 0;
+    }
+
+    // vec kernel instantiations exist for these (type, dk, dv) combinations only
+    const int64_t dk = op->src[1]->ne[0];
+    const int64_t dv = op->src[2]->ne[0];
+
+    const bool dk_dv_ok = (dk == 32  && dv == 32)  ||
+                          (dk == 64  && dv == 64)  ||
+                          (dk == 96  && dv == 96)  ||
+                          (dk == 128 && dv == 128) ||
+                          (dk == 192 && dv == 128) ||
+                          (dk == 192 && dv == 192) ||
+                          (dk == 256 && dv == 256) ||
+                          (dk == 320 && dv == 256) ||
+                          (dk == 512 && dv == 512) ||
+                          (dk == 576 && dv == 512);
+
+    if (!dk_dv_ok) {
+        return 0;
+    }
+
+    switch (op->src[1]->type) {
+        case GGML_TYPE_F16:
+        case GGML_TYPE_BF16:
+        case GGML_TYPE_F32:
+        case GGML_TYPE_Q4_0:
+        case GGML_TYPE_Q4_1:
+        case GGML_TYPE_Q5_0:
+        case GGML_TYPE_Q5_1:
+        case GGML_TYPE_Q8_0:
+            break;
+        default:
+            return 0;
+    }
+
+    return n_kv_max;
+}
+
 // in some models (e.g. MLA-based), V is a view of K (the first ne20 elements of each K row);
 // the dequantized V is then a view of the dequantized K and does not need its own dequant or scratch
 // - ref: https://github.com/ggml-org/llama.cpp/pull/13435
@@ -3027,6 +3086,24 @@ size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) {
    return k_size + v_size;
 }

+// size of the sparse index lists: one list of KV indices per mask row,
+// padded with -1 up to a multiple of OP_FLASH_ATTN_EXT_VEC_NCPSG
+size_t ggml_metal_op_flash_attn_ext_extra_idx(const ggml_tensor * op) {
+    assert(op->op == GGML_OP_FLASH_ATTN_EXT);
+
+    GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne);
+
+    const int n_kv_max = ggml_metal_op_flash_attn_ext_n_kv_max_sparse(op);
+
+    if (n_kv_max <= 0) {
+        return 0;
+    }
+
+    const int n_kv_max_padded = GGML_PAD(n_kv_max, OP_FLASH_ATTN_EXT_VEC_NCPSG);
+
+    return GGML_PAD(sizeof(int32_t)*(size_t) n_kv_max_padded*ne31*ne32*ne33, 16);
+}
+
 int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
    ggml_tensor * op = ctx->node(idx);

@@ -3104,7 +3181,16 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
    ggml_metal_buffer_id bid_kv_f16 = bid_tmp;
    bid_kv_f16.offs += ggml_metal_op_flash_attn_ext_extra_tmp(op);

-    const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op);
+    // sparse path: gather the finite mask entries into index lists and run the vec kernels over them
+    const int n_kv_max_sparse = ggml_metal_op_flash_attn_ext_n_kv_max_sparse(op);
+    const bool use_sparse = n_kv_max_sparse > 0;
+    const int n_kv_max_padded = use_sparse ? GGML_PAD(n_kv_max_sparse, OP_FLASH_ATTN_EXT_VEC_NCPSG) : 0;
+
+    // the vec kernels dequantize the KV inline; no need for the F16 dequant pass in the sparse path
+    const bool use_kv_f16 = !use_sparse && ggml_metal_op_flash_attn_ext_use_kv_f16(op);
+
+    ggml_metal_buffer_id bid_idx = bid_kv_f16;
+    bid_idx.offs += ggml_metal_op_flash_attn_ext_extra_kv_f16(op);

    ggml_metal_buffer_id bid_k = bid_src1;
    ggml_metal_buffer_id bid_v = bid_src2;
@@ -3206,7 +3292,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
        }
    }

-    if (!ggml_metal_op_flash_attn_ext_use_vec(op)) {
+    if (!use_sparse && !ggml_metal_op_flash_attn_ext_use_vec(op)) {
        // half8x8 kernel
        const int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroup
        const int ncpsg = OP_FLASH_ATTN_EXT_NCPSG; // cache values per simdgroup
@@ -3378,13 +3464,18 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
 #undef FATTN_SMEM
    } else {
        // half4x4 kernel
-        auto cfg = ggml_metal_tuning::fa_vec_pick(
-                props_dev->device_id,
-                props_dev->gpu_family,
-                (int) op->src[1]->type,
-                (int) ne00, (int) ne20,   // dk, dv (ne00 == dk for FA)
-                ne11, ne01);
-        int nqptg = cfg.Q;                             // queries per threadgroup
+        // sparse: the index lists are per query row, so a threadgroup can share KV with Q == 1 only
+        auto cfg = use_sparse
+                ? ggml_metal_tuning::fa_vec_baseline_cfg((int) ne00, (int) ne20)
+                : ggml_metal_tuning::fa_vec_pick(
+                          props_dev->device_id,
+                          props_dev->gpu_family,
+                          (int) op->src[1]->type,
+                          (int) ne00, (int) ne20,   // dk, dv (ne00 == dk for FA)
+                          ne11, ne01);
+
+        int nqptg = cfg.Q; // queries per threadgroup
+
        const int ncpsg = OP_FLASH_ATTN_EXT_VEC_NCPSG; // cache values per simdgroup !! sync with kernel template arguments !!
        const int nhptg = 1;                           // heads per threadgroup

@@ -3394,7 +3485,39 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {

        bool need_sync = false;

-        const bool has_kvpad = ne11 % ncpsg != 0;
+        const bool has_kvpad = !use_sparse && ne11 % ncpsg != 0;
+
+        if (use_sparse) {
+            assert(ggml_metal_op_flash_attn_ext_extra_idx(op) != 0);
+
+            GGML_ASSERT(ne30 == ne11);
+
+            ggml_metal_kargs_flash_attn_ext_vec_idx args0 = {
+                /*.ne30              =*/ ne30,
+                /*.ne31              =*/ ne31,
+                /*.ne32              =*/ ne32,
+                /*.ne33              =*/ ne33,
+                /*.nb31              =*/ nb31,
+                /*.nb32              =*/ nb32,
+                /*.nb33              =*/ nb33,
+                /*.n_kv_max          =*/ n_kv_max_sparse,
+                /*.n_kv_max_padded   =*/ n_kv_max_padded,
+            };
+
+            auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(lib, op);
+
+            ggml_metal_encoder_set_pipeline(enc, pipeline0);
+            ggml_metal_encoder_set_bytes   (enc, &args0, sizeof(args0), 0);
+            ggml_metal_encoder_set_buffer  (enc, bid_src3, 1);
+            ggml_metal_encoder_set_buffer  (enc, bid_idx,  2);
+
+            int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256);
+            nth = std::max(32, (nth/32)*32);
+
+            ggml_metal_encoder_dispatch_threadgroups(enc, ne31, ne32, ne33, nth, 1, 1);
+
+            need_sync = true;
+        }

        if (has_kvpad) {
            assert(ggml_metal_op_flash_attn_ext_extra_pad(op) != 0);
@@ -3455,11 +3578,26 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
        // workgroups
        // each workgroup handles nsg*nkpsg cache values
        int32_t nwg = 1;
-        if (false) {
-            // for small KV caches, we could launch a single workgroup and write the results directly to dst/
-            // however, this does not lead to significant improvement, so disabled
-            nwg = 1;
-            nsg = 4;
+        if (use_sparse) {
+            if (ne01 > 32) {
+                // large sparse batch
+                nwg = 1;
+                nsg = 1;
+                if (n_kv_max_padded == 640) {
+                    nsg = 4; // 640 % (4*32) == 0
+                } else {
+                    while (2*nwg*nsg*ncpsg < n_kv_max_padded && nsg < 4) {
+                        nsg *= 2;
+                    }
+                }
+            } else {
+                // small sparse batch
+                nwg = 32;
+                nsg = 1;
+                while (2*nwg*nsg*ncpsg < n_kv_max_padded && nsg < 4) {
+                    nsg *= 2;
+                }
+            }
        } else {
            nwg = 32;
            nsg = 1;
@@ -3484,7 +3622,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
            /*.nb01          =*/ nb01,
            /*.nb02          =*/ nb02,
            /*.nb03          =*/ nb03,
-            /*.ne11          =*/ ne11,
+            /*.ne11          =*/ use_sparse ? n_kv_max_padded : ne11,
            /*.ne_12_2       =*/ ne12,
            /*.ne_12_3       =*/ ne13,
            /*.ns10          =*/ ns10,
@@ -3510,9 +3648,10 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
            /*.m1            =*/ m1,
            /*.n_head_log2   =*/ n_head_log2,
            /*.logit_softcap =*/ logit_softcap,
+            /*.n_kv_max_padded =*/ n_kv_max_padded,
        };

-        auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20);
+        auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20);

        GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));

@@ -3523,6 +3662,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
        ggml_metal_encoder_set_buffer  (enc, bid_v,    3);
        ggml_metal_encoder_set_buffer  (enc, bid_src3, 4);
        ggml_metal_encoder_set_buffer  (enc, bid_src4, 5);
+        ggml_metal_encoder_set_buffer  (enc, use_sparse ? bid_idx : bid_src0, 8);

        const size_t smem = FATTN_SMEM(nsg);

@@ -3530,8 +3670,6 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
        GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);

        if (nwg == 1) {
-            assert(ggml_metal_op_flash_attn_ext_extra_tmp(op) == 0);
-
            // using 1 workgroup -> write the result directly into dst
            ggml_metal_encoder_set_buffer(enc, bid_pad, 6);
            ggml_metal_encoder_set_buffer(enc, bid_dst, 7);
@@ -43,6 +43,7 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op);
 size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op);
 size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op);
 size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op);
+size_t ggml_metal_op_flash_attn_ext_extra_idx(const struct ggml_tensor * op);

 int ggml_metal_op_concat            (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_repeat            (ggml_metal_op_t ctx, int idx);
@@ -1248,6 +1248,153 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 0 }, { 4, 4 } },
    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 1 }, { 4, 1 } },
    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 2 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 3 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 64, 64, 2, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 128, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 128, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 192, 128, 3, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } },
    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } },
    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } },
    { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } },
@@ -1525,6 +1672,178 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
    { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 2, 1 }, { 4, 2 } },
    { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 2, 2 }, { 4, 2 } },
    { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 3, 1 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 1, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 3, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 3, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 3, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 3, 3 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 320, 256, 3, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, 1, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 96, 96, 1, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 2, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 3, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 2, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 3, 3 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 512, 512, 2, 0 }, { 4, 1 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 512, 512, 2, 4 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 512, 512, 3, 2 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, 2, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 320, 256, 1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 512, 512, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, 1, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, 3, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 2, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, 3, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 3, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 3, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 3, 3 }, { 1, 4 } },
    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
    { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
@@ -1826,6 +2145,151 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 2, 2 }, { 4, 4 } },
    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 2, 3 }, { 4, 2 } },
    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 3, 1 }, { 4, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 128, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 128, 3, 3 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 4 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 2 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 3, 3 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 4, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 4 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 4 }, { 1, 2 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } },
+    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } },
    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
    { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 4 }, { 2, 4 } },
@@ -232,6 +232,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
                res += ggml_metal_op_flash_attn_ext_extra_blk(tensor);
                res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor);
                res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor);
+                res += ggml_metal_op_flash_attn_ext_extra_idx(tensor);
            } break;
        case GGML_OP_CUMSUM:
        case GGML_OP_ARGSORT:
@@ -1071,6 +1071,112 @@ constant int32_t FC_flash_attn_ext_vec_ns10 [[function_constant(FC_FLASH_ATTN_EX
 constant int32_t FC_flash_attn_ext_vec_ns20 [[function_constant(FC_FLASH_ATTN_EXT_VEC + 21)]];
 constant int32_t FC_flash_attn_ext_vec_nsg  [[function_constant(FC_FLASH_ATTN_EXT_VEC + 22)]];
 constant int32_t FC_flash_attn_ext_vec_nwg  [[function_constant(FC_FLASH_ATTN_EXT_VEC + 23)]];
+constant bool    FC_flash_attn_ext_vec_has_sparse [[function_constant(FC_FLASH_ATTN_EXT_VEC + 5)]];
+
+// compress the finite entries of each KQ mask row into a list of KV indices (ascending order),
+// padded with -1 up to n_kv_max_padded (a multiple of OP_FLASH_ATTN_EXT_VEC_NCPSG)
+// one threadgroup per mask row; the mask remains the single source of truth for the values
+kernel void kernel_flash_attn_ext_vec_idx(
+        constant ggml_metal_kargs_flash_attn_ext_vec_idx & args,
+        device const half * mask,
+        device       int  * idx,
+        uint3   tgpig[[threadgroup_position_in_grid]],
+        ushort  tiitg[[thread_index_in_threadgroup]],
+        ushort3 ntg[[threads_per_threadgroup]]) {
+    constexpr short NW = N_SIMDWIDTH;
+    constexpr short NLOCAL = 32; // max finite positions kept in registers per thread
+
+    const int i1 = tgpig[0];
+    const int i2 = tgpig[1];
+    const int i3 = tgpig[2];
+
+    device const half * pm  = (device const half *) ((device const char *) mask + i1*args.nb31 + i2*args.nb32 + i3*args.nb33);
+    device int * pidx = idx + (((int64_t)i3*args.ne32 + i2)*args.ne31 + i1)*args.n_kv_max_padded;
+
+    const int n  = args.ne30;
+    const int q  = n/ntg.x;
+    const int r  = n%ntg.x;
+
+    // each thread handles a contiguous slice of the mask row
+    const int r0 = q*tiitg + min((int) tiitg, r);
+    const int r1 = r0 + q + (tiitg < r ? 1 : 0);
+
+    // count the finite entries in the slice and keep their positions in registers (single mask read)
+    int cnt = 0;  // total finite entries in the slice
+    int nloc = 0; // finite entries kept in registers
+    int local[NLOCAL];
+    for (int i = r0; i < r1; ++i) {
+        if (isfinite((float) pm[i])) {
+            if (nloc < NLOCAL) {
+                local[nloc] = i;
+                nloc++;
+            }
+            cnt++;
+        }
+    }
+
+    const short sgitg = tiitg/NW;
+    const short tiisg = tiitg%NW;
+
+    threadgroup int tcount[8];
+
+    // simd_sum is a collective: all lanes must evaluate it
+    const int sg_sum = simd_sum(cnt);
+    if (tiisg == 0) {
+        tcount[sgitg] = sg_sum;
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    int total = 0;
+    for (short s = 0; s < ntg.x/NW; ++s) {
+        total += tcount[s];
+    }
+
+    // base offset of this thread's slice in the output list (exclusive scan within the simdgroup)
+    int sg_base = 0;
+    for (short s = 0; s < sgitg; ++s) {
+        sg_base += tcount[s];
+    }
+
+    // exclusive prefix scan of the per-thread counts within the simdgroup
+    int incl = cnt;
+    for (int d = 1; d < NW; d <<= 1) {
+        const int v = simd_shuffle_up(incl, d);
+        if (tiisg >= d) {
+            incl += v;
+        }
+    }
+    const int base = sg_base + (incl - cnt);
+
+    // write the finite positions in order; if the hint is violated, keep only the first n_kv_max entries
+    int j = 0;
+    for (; j < nloc && base + j < args.n_kv_max; ++j) {
+        pidx[base + j] = local[j];
+    }
+
+    // a dense mask may have more than NLOCAL finite entries in a slice; re-read the mask to write the rest
+    if (cnt > nloc && base + nloc < args.n_kv_max) {
+        int j2 = 0;
+        for (int i = r0; i < r1; ++i) {
+            if (isfinite((float) pm[i])) {
+                if (j2 >= nloc) {
+                    pidx[base + j2] = i;
+                }
+                j2++;
+                if (base + j2 >= args.n_kv_max) {
+                    break;
+                }
+            }
+        }
+    }
+
+    // pad the tail of the list with -1
+    const int count = min(total, args.n_kv_max);
+    for (int i = count + tiitg; i < args.n_kv_max_padded; i += ntg.x) {
+        pidx[i] = -1;
+    }
+}

 template<
    typename q4_t,  // query types in shared memory
@@ -1091,6 +1197,7 @@ template<
    short NE = 4,   // head elements per thread
    short Q  = OP_FLASH_ATTN_EXT_VEC_NQPSG,  // queries per threadgroup
    short C  = OP_FLASH_ATTN_EXT_VEC_NCPSG>  // cache items per threadgroup
+
 kernel void kernel_flash_attn_ext_vec(
        constant ggml_metal_kargs_flash_attn_ext_vec & args,
        device const char * q,
@@ -1100,6 +1207,7 @@ kernel void kernel_flash_attn_ext_vec(
        device const char * sinks,
        device const char * pad,
        device       char * dst,
+        device const char * idx,
        threadgroup  half * shmem_f16 [[threadgroup(0)]],
        uint3   tgpig[[threadgroup_position_in_grid]],
        ushort  tiisg[[thread_index_in_simdgroup]],
@@ -1137,8 +1245,8 @@ kernel void kernel_flash_attn_ext_vec(

  //const short T = PK + NSG*SH; // shared memory size per query in (half)

-  //threadgroup q_t   * sq  = (threadgroup q_t   *) (shmem_f16 +                            0*PK); // holds the query data
-    threadgroup q4_t  * sq4 = (threadgroup q4_t  *) (shmem_f16 +                            0*PK); // same as above but in q4_t
+  //threadgroup q_t   * sq  = (threadgroup q_t   *) (shmem_f16 +                          0*PK); // holds the query data
+    threadgroup q4_t  * sq4 = (threadgroup q4_t  *) (shmem_f16 +                          0*PK); // same as above but in q4_t
    threadgroup s_t   * ss  = (threadgroup s_t   *) (shmem_f16 +   sgitg*SH         + Q*NSG*PK); // scratch buffer for attention
    threadgroup s4_t  * ss4 = (threadgroup s4_t  *) (shmem_f16 +   sgitg*SH         + Q*NSG*PK); // same as above but in s4_t
    threadgroup half  * sm  = (threadgroup half  *) (shmem_f16 +   sgitg*SH + 2*Q*C + Q*NSG*PK); // scratch buffer for mask
@@ -1207,6 +1315,14 @@ kernel void kernel_flash_attn_ext_vec(
        // pointer to the mask
        device const half * pm_base = (device const half *) (mask + iq1*Q*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33);

+        // sparse indices: the list of finite mask entries per query row
+        // the sparse path requires Q == 1 (enforced by the host)
+        device const int * pidx = nullptr;
+        if (FC_flash_attn_ext_vec_has_sparse) {
+            pidx = (device const int *) idx +
+                ((int64_t)(iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*args.ne31*args.n_kv_max_padded + (iq1%args.ne31)*args.n_kv_max_padded;
+        }
+
        float slope = 1.0f;

        // ALiBi
@@ -1265,11 +1381,22 @@ kernel void kernel_flash_attn_ext_vec(
            }

            if (FC_flash_attn_ext_vec_has_mask) {
-                FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
-                    if ((iq1*Q + qq) < args.ne01) {
-                        sm[qq*C + tiisg] = pm[qq][ic + tiisg];
-                    } else {
-                        sm[qq*C + tiisg] = -MAXHALF;
+                if (FC_flash_attn_ext_vec_has_sparse) {
+                    FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
+                        const int i11 = pidx[ic + tiisg];
+                        if ((iq1*Q + qq) < args.ne01 && i11 >= 0) {
+                            sm[qq*C + tiisg] = pm[qq][i11];
+                        } else {
+                            sm[qq*C + tiisg] = -MAXHALF;
+                        }
+                    }
+                } else {
+                    FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
+                        if ((iq1*Q + qq) < args.ne01) {
+                            sm[qq*C + tiisg] = pm[qq][ic + tiisg];
+                        } else {
+                            sm[qq*C + tiisg] = -MAXHALF;
+                        }
                    }
                }
            } else {
@@ -1280,6 +1407,7 @@ kernel void kernel_flash_attn_ext_vec(
                }
            }

+            // skip -INF mask
            {
                bool any_finite = false;
                FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
@@ -1294,9 +1422,13 @@ kernel void kernel_flash_attn_ext_vec(

            // Q*K^T
            {
-                device      const k4_t * pk4 = (device const k4_t *) (k + ic*args.nb11);
+                device      const k4_t * pk4 = nullptr;

-                pk4 += ty*NS10/4 + tx;
+                if (!FC_flash_attn_ext_vec_has_sparse) {
+                    pk4 = (device const k4_t *) (k + ic*args.nb11);
+
+                    pk4 += ty*NS10/4 + tx;
+                }

                qk_t mqk[Q][C/NE];
                FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
@@ -1307,7 +1439,35 @@ kernel void kernel_flash_attn_ext_vec(

                // each simdgroup processes Q queries and NE (NW/NL) cache elements
                FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {
-                    if (is_same<kd4_t, k4_t>::value) {
+                    if (FC_flash_attn_ext_vec_has_sparse) {
+                        // the KV rows are gathered from the index list; -1 entries are padding
+                        const int i11 = pidx[ic + NE*cc + ty];
+                        if (i11 >= 0) {
+                            if (is_same<kd4_t, k4_t>::value) {
+                                device const k4_t * pk4s = (device const k4_t *) (k + i11*args.nb11) + tx;
+                                FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) {
+                                    const k4_t k_elem = pk4s[ii*NL];
+                                    FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
+                                        mqk[qq][cc] += dot((float4) k_elem, (float4) sq4[qq*PK4 + ii*NL + tx]);
+                                    }
+                                }
+                            } else {
+                                device const kd4_t * pk = (device const kd4_t *) (k + i11*args.nb11);
+
+                                k4_t mk;
+
+                                FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) {
+                                    const short i = ii*NL + tx;
+
+                                    deq_k_t4(pk + i/nl_k, i%nl_k, mk);
+
+                                    FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
+                                        mqk[qq][cc] += dot((float4) mk, (float4) sq4[qq*PK4 + i]);
+                                    }
+                                }
+                            }
+                        }
+                    } else if (is_same<kd4_t, k4_t>::value) {
                        FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) {
                            const k4_t k_elem = pk4[cc*NE*NS10/4 + ii*NL];
                            FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
@@ -1422,7 +1582,40 @@ kernel void kernel_flash_attn_ext_vec(
                    }
                }

-                if (is_same<vd4_t, v4_t>::value) {
+                if (FC_flash_attn_ext_vec_has_sparse) {
+                    FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {
+                        // the KV rows are gathered from the index list; -1 entries are padding
+                        const int i11 = pidx[ic + NE*cc + ty];
+                        if (i11 >= 0) {
+                            if (is_same<vd4_t, v4_t>::value) {
+                                device const v4_t * pv4 = (device const v4_t *) (v + i11*args.nb21);
+
+                                pv4 += tx;
+
+                                FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {
+                                    const v4_t v_elem = pv4[ii*NL];
+                                    FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
+                                        lo[qq][ii] += o4_t(float4(v_elem)*float4(ss[qq*C + cc*NE + ty]));
+                                    }
+                                }
+                            } else {
+                                device const vd4_t * pv4 = (device const vd4_t *) (v + i11*args.nb21);
+
+                                FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {
+                                    const short i = ii*NL + tx;
+
+                                    v4_t mv;
+
+                                    deq_v_t4(pv4 + i/nl_v, i%nl_v, mv);
+
+                                    FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
+                                        lo[qq][ii] += o4_t(float4(mv)*float4(ss[qq*C + cc*NE + ty]));
+                                    }
+                                }
+                            }
+                        }
+                    }
+                } else if (is_same<vd4_t, v4_t>::value) {
                    device const v4_t * pv4 = (device const v4_t *) (v + ic*args.nb21);

                    pv4 += ty*NS20/4 + tx;
@@ -75,7 +75,6 @@ if (MUSAToolkit_FOUND)
    endif()

    add_compile_definitions(GGML_USE_MUSA)
-    add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})

    if (GGML_MUSA_GRAPHS)
        add_compile_definitions(GGML_MUSA_GRAPHS)
@@ -85,6 +85,7 @@ set(GGML_OPENCL_KERNELS
    mul_mv_f16_f32_1row
    mul_mv_f16_f32_l4
    mul_mv_f16_f32
+    mul_mv_f16_f32_mrow
    mul_mv_f32_f32
    mul_mv_q1_0_f32
    mul_mv_q1_0_f32_flat
@@ -180,9 +181,14 @@ set(GGML_OPENCL_KERNELS
    gemv_noshuffle_q8_0_f32
    gemm_noshuffle_q8_0_f32
    gemv_noshuffle_q4_k_f32
+    gemv_noshuffle_q4_k_f32_o4
+    gemv_noshuffle_q4_k_f32_tiled
    gemm_noshuffle_q4_k_f32
    gemv_noshuffle_q6_k_f32
+    gemv_noshuffle_q6_k_f32_o4
+    gemv_noshuffle_q6_k_f32_tiled
    gemm_noshuffle_q6_k_f32
+    gemm_noshuffle_q6_k_f32_tiled
    gemv_noshuffle_q5_k_f32
    gemm_noshuffle_q5_k_f32
    mul
@@ -216,6 +222,7 @@ set(GGML_OPENCL_KERNELS
    exp
    expm1
    abs
+    unary_ext
    softplus
    pad
    repeat
@@ -232,7 +239,7 @@ set(GGML_OPENCL_KERNELS
 )

 if (GGML_OPENCL_USE_ADRENO_KERNELS)
-    list(APPEND GGML_OPENCL_KERNELS gemm_xmem_f16_f32_os8)
+    list(APPEND GGML_OPENCL_KERNELS gemm_xmem_f16_f32_os8 sdpa_xmem_f32_f16_os8)
 endif ()

 foreach (K ${GGML_OPENCL_KERNELS})
@@ -1,56 +1,66 @@
-kernel void kernel_concat_f32(
-    global  const char * src0,
-    ulong                offset0,
-    global  const char * src1,
-    ulong                offset1,
-    global        char * dst,
-    ulong                offsetd,
-    int             ne00,
-    int             ne01,
-    int             ne02,
-    int             ne03,
-    ulong           nb00,
-    ulong           nb01,
-    ulong           nb02,
-    ulong           nb03,
-    ulong           nb10,
-    ulong           nb11,
-    ulong           nb12,
-    ulong           nb13,
-    int             ne0,
-    ulong           nb0,
-    ulong           nb1,
-    ulong           nb2,
-    ulong           nb3,
-    int             dim
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
+// concat is a pure copy, so the kernels are keyed by element byte size
+// (1/2/4/8) rather than logical type, matching the CUDA backend.

-    const int i3 = get_group_id(2);
-    const int i2 = get_group_id(1);
-    const int i1 = get_group_id(0);
-
-    int o[4] = {0, 0, 0, 0};
-    o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));
-
-    global const float * x;
-
-    for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
-        if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
-            x = (global const float *)(src0 + (i3       )*nb03 + (i2       )*nb02 + (i1       )*nb01 + (i0       )*nb00);
-        } else {
-            x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
-        }
-
-        global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
-
-        *y = *x;
-    }
+#define KERNEL_CONCAT(SUFFIX, T)                                                     \
+kernel void kernel_concat_##SUFFIX(                                                  \
+    global  const char * src0,                                                       \
+    ulong                offset0,                                                    \
+    global  const char * src1,                                                       \
+    ulong                offset1,                                                    \
+    global        char * dst,                                                        \
+    ulong                offsetd,                                                    \
+    int             ne00,                                                            \
+    int             ne01,                                                            \
+    int             ne02,                                                            \
+    int             ne03,                                                            \
+    ulong           nb00,                                                            \
+    ulong           nb01,                                                            \
+    ulong           nb02,                                                            \
+    ulong           nb03,                                                            \
+    ulong           nb10,                                                            \
+    ulong           nb11,                                                            \
+    ulong           nb12,                                                            \
+    ulong           nb13,                                                            \
+    int             ne0,                                                             \
+    ulong           nb0,                                                             \
+    ulong           nb1,                                                             \
+    ulong           nb2,                                                             \
+    ulong           nb3,                                                             \
+    int             dim                                                              \
+) {                                                                                  \
+    src0 = src0 + offset0;                                                           \
+    src1 = src1 + offset1;                                                           \
+    dst  = dst  + offsetd;                                                           \
+                                                                                     \
+    const int i3 = get_group_id(2);                                                  \
+    const int i2 = get_group_id(1);                                                  \
+    const int i1 = get_group_id(0);                                                  \
+                                                                                     \
+    int o[4] = {0, 0, 0, 0};                                                         \
+    o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));         \
+                                                                                     \
+    global const T * x;                                                              \
+                                                                                     \
+    for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {              \
+        if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {                      \
+            x = (global const T *)(src0 + (i3       )*nb03 + (i2       )*nb02 + (i1       )*nb01 + (i0       )*nb00); \
+        } else {                                                                     \
+            x = (global const T *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); \
+        }                                                                            \
+                                                                                     \
+        global T * y = (global T *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);        \
+                                                                                     \
+        *y = *x;                                                                     \
+    }                                                                                \
 }

-kernel void kernel_concat_f32_pack(
+KERNEL_CONCAT(b1, char)
+KERNEL_CONCAT(b2, short)
+KERNEL_CONCAT(b4, int)
+KERNEL_CONCAT(b8, long)
+
+// packed variant for the common dim==0, small-ne0 case (4-byte elements only).
+kernel void kernel_concat_b4_pack(
    global  const char * src0,
    ulong                offset0,
    global  const char * src1,
@@ -104,14 +114,14 @@ kernel void kernel_concat_f32_pack(
    o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));

    for (int i0 = lane; i0 < ne0; i0 += tpr) {
-        global const float * x;
+        global const int * x;
        if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
-            x = (global const float *)(src0 + (i3       )*nb03 + (i2       )*nb02 + (i1       )*nb01 + (i0       )*nb00);
+            x = (global const int *)(src0 + (i3       )*nb03 + (i2       )*nb02 + (i1       )*nb01 + (i0       )*nb00);
        } else {
-            x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
+            x = (global const int *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
        }

-        global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+        global int * y = (global int *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);

        *y = *x;
    }
@@ -286,3 +286,28 @@ kernel void kernel_cpy_i32_i32(
        dst_data[i00] = src[0];
    }
 }
+
+// Contiguous f32 copy, one work item per float4 over the whole tensor. The kernels above map
+// one workgroup to each row, which leaves a tensor with few long rows on a single compute unit.
+// vload4/vstore4 rather than a float4 cast: these buffers carry an arbitrary 4-byte view offset.
+kernel void kernel_cpy_f32_f32_flat(
+        global float * src0,
+        ulong offset0,
+        global float * dst,
+        ulong offsetd,
+        ulong ne,
+        ulong n4
+) {
+    src0 = (global float*)((global char*)src0 + offset0);
+    dst  = (global float*)((global char*)dst  + offsetd);
+
+    const ulong i = get_global_id(0);
+
+    if (i < n4) {
+        vstore4(vload4(i, src0), i, dst);
+    } else if (i == n4) {
+        for (ulong t = n4 * 4; t < ne; ++t) {
+            dst[t] = src0[t];
+        }
+    }
+}
@@ -1110,6 +1110,78 @@ kernel void kernel_restore_block_q4_k_trans4_ns(
    }
 }

+//------------------------------------------------------------------------------
+// kernel_convert_block_q4_k_tiled_ns
+//
+// Tiled-wide layout for the long-vocab q4_K lm_head/embed GEMV (decode path).
+// Mirror of kernel_convert_block_q6_k_tiled_ns: recovers each weight's 4-bit
+// code in CANONICAL ggml element order (e in [0,256)) and re-packs into 32 uints
+// (8 codes/uint), stored TILED by 64 output rows so the matching GEMV
+// (gemv_noshuffle_q4_k_f32_tiled) coalesces every weight load. The 12-byte
+// packed scale block `s` and d/dm are stored per (row, K-block) tiled; the GEMV
+// re-derives the 8 (scale,min) pairs via get_scale_min_k4, exactly like the o4
+// kernel. Both ends owned here -> correct by construction vs the reference q4_K
+// dequant. Requires ne01 % 64 == 0 (gated host-side). Buffer sizes identical to
+// the trans4_ns layout.
+//
+//   q  uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk+sb)*8 + g)*64 + rit
+//   s  (12 bytes) of (r, sb):                  idx = (rt*ne00_blk+sb)*64 + rit, *12
+//   d/dm (half)  of (r, sb):                   idx = (rt*ne00_blk+sb)*64 + rit
+//   where rt = r/64, rit = r%64.
+//------------------------------------------------------------------------------
+kernel void kernel_convert_block_q4_k_tiled_ns(
+    __global struct block_q4_K * src0,
+    __global uint  * dst_q,    // 32 uints / superblock (4-bit codes, 8 codes/uint)
+    __global half  * dst_d,    // 1 half  / superblock
+    __global half  * dst_dm,   // 1 half  / superblock
+    __global uchar * dst_s,    // K_SCALE_SIZE (12) bytes / superblock
+    uint ne00,
+    uint ne01
+) {
+    uint i00 = get_global_id(1);   // K-block index (superblock along ne00)
+    uint i01 = get_global_id(0);   // output row index (along ne01)
+    uint i02 = get_global_id(2);   // batch
+
+    uint ne00_blk = ne00 / QK_K;
+
+    uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
+    __global struct block_q4_K * b = src0 + src_blk_offset;
+
+    uint rt  = i01 / 64;
+    uint rit = i01 % 64;
+    uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00;
+
+    // --- recover canonical 4-bit codes in e-order, pack 8 codes/uint ---
+    uint qw[32] = {0};
+    for (uint e = 0; e < 256; ++e) {
+        uint g    = e >> 6;           // group 0..3 (q advances 32 bytes/group)
+        uint within = e & 63u;
+        uint hlf  = within >> 5;      // 0 = low nibble, 1 = high nibble
+        uint l    = within & 31u;     // 0..31
+        uchar byte = b->q[g * 32u + l];
+        uint code = (hlf == 0u) ? (uint)(byte & 0x0F) : (uint)(byte >> 4);
+        qw[e >> 3] |= code << ((e & 7u) * 4u);
+    }
+
+    for (uint gr = 0; gr < 8; ++gr) {
+        uint base = (tile_blk * 8u + gr) * 64u + rit;   // uint4 index
+        dst_q[base * 4u + 0u] = qw[gr * 4u + 0u];
+        dst_q[base * 4u + 1u] = qw[gr * 4u + 1u];
+        dst_q[base * 4u + 2u] = qw[gr * 4u + 2u];
+        dst_q[base * 4u + 3u] = qw[gr * 4u + 3u];
+    }
+
+    // packed scales (12 bytes), tiled per (row, block)
+    __global uchar * s_dst = dst_s + (tile_blk * 64u + rit) * K_SCALE_SIZE;
+    #pragma unroll
+    for (int i = 0; i < K_SCALE_SIZE; ++i) {
+        s_dst[i] = b->s[i];
+    }
+
+    dst_d [tile_blk * 64u + rit] = b->d;
+    dst_dm[tile_blk * 64u + rit] = b->dm;
+}
+
 kernel void kernel_convert_block_q5_k_trans4_ns(
    __global struct block_q5_K * src0,
    __global uint  * dst_qs,
@@ -1494,6 +1566,105 @@ kernel void kernel_restore_block_mxfp4_trans(
    b->e = src_e[src_blk_offset];
 }

+//------------------------------------------------------------------------------
+// kernel_convert_block_q6_k_tiled_ns
+//
+// Tiled-wide layout for the long-vocab q6_K lm_head/embed GEMV (decode path).
+// Unlike *_trans4_ns (which mirrors the bit-interleave the legacy 2-output GEMV
+// consumes), this kernel is correct-by-construction against the CANONICAL ggml
+// q6_K dequant: it recovers each weight's 6-bit code in element order e in
+// [0,256), then re-packs low-4-bits into 32 uints (8 codes/uint) and high-2-bits
+// into 16 uints (16 codes/uint). The matching GEMV (gemv_noshuffle_q6_k_f32_tiled)
+// unpacks the same order, so both ends are owned here.
+//
+// Storage is TILED by 64 output rows so the GEMV's 64-thread tile coalesces:
+//   ql uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk + sb)*8 + g)*64 + rit
+//   qh uint4 granule g:                         idx = ((rt*ne00_blk + sb)*4 + g)*64 + rit
+//   scales (char16) of (r, sb):                 idx = (rt*ne00_blk + sb)*64 + rit
+//   d (half) of (r, sb):                        idx = (rt*ne00_blk + sb)*64 + rit
+// where rt = r/64, rit = r%64. Requires ne01 % 64 == 0 (gated host-side).
+// Buffer sizes are byte-identical to the trans4_ns layout.
+//------------------------------------------------------------------------------
+kernel void kernel_convert_block_q6_k_tiled_ns(
+    __global struct block_q6_K * src0,
+    __global uint  * dst_ql,   // 32 uints / superblock (low 4 bits, 8 codes/uint)
+    __global uint  * dst_qh,   // 16 uints / superblock (high 2 bits, 16 codes/uint)
+    __global half  * dst_d,    // 1 half  / superblock
+    __global char  * dst_s,    // 16 chars/ superblock
+    uint ne00,
+    uint ne01
+) {
+    uint i00 = get_global_id(1);   // K-block index (superblock along ne00)
+    uint i01 = get_global_id(0);   // output row index (along ne01)
+    uint i02 = get_global_id(2);   // batch
+
+    uint ne00_blk = ne00 / QK_K;
+
+    // Source block: row-major over (i02, i01, i00).
+    uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
+    __global struct block_q6_K * b = src0 + src_blk_offset;
+
+    uint rt  = i01 / 64;
+    uint rit = i01 % 64;
+    uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00;  // tile-major (row-tile, K-block)
+
+    // --- recover canonical 6-bit codes, pack into ql (4b) + qh (2b) in e-order ---
+    // 32 ql-uints (8 low-nibbles each) + 16 qh-uints (16 2-bit slots each).
+    uint qlw[32] = {0};
+    uint qhw[16] = {0};
+
+    for (uint e = 0; e < 256; ++e) {
+        uint n   = (e >= 128) ? 1u : 0u;       // which 128-half
+        uint within = e - n * 128u;
+        uint q   = within / 32u;               // quadrant 0..3
+        uint l   = within % 32u;               // 0..31
+
+        uint off_ql = n * 64u;                 // raw ql byte base for this half
+        uint off_qh = n * 32u;                 // raw qh byte base for this half
+
+        uchar low4;
+        uchar qlb0 = b->ql[off_ql + l];
+        uchar qlb1 = b->ql[off_ql + l + 32];
+        if (q == 0)      low4 = qlb0 & 0x0F;
+        else if (q == 1) low4 = qlb1 & 0x0F;
+        else if (q == 2) low4 = (qlb0 >> 4) & 0x0F;
+        else             low4 = (qlb1 >> 4) & 0x0F;
+
+        uchar hi2 = (b->qh[off_qh + l] >> (q * 2u)) & 0x03;
+
+        // pack low4 (e-order): uint e/8, nibble (e%8)
+        qlw[e >> 3] |= ((uint)low4) << ((e & 7u) * 4u);
+        // pack hi2 (e-order): uint e/16, 2-bit slot (e%16)
+        qhw[e >> 4] |= ((uint)hi2)  << ((e & 15u) * 2u);
+    }
+
+    // --- write tiled ---
+    for (uint g = 0; g < 8; ++g) {
+        uint base = (tile_blk * 8u + g) * 64u + rit;   // uint4 index
+        dst_ql[base * 4u + 0u] = qlw[g * 4u + 0u];
+        dst_ql[base * 4u + 1u] = qlw[g * 4u + 1u];
+        dst_ql[base * 4u + 2u] = qlw[g * 4u + 2u];
+        dst_ql[base * 4u + 3u] = qlw[g * 4u + 3u];
+    }
+    for (uint g = 0; g < 4; ++g) {
+        uint base = (tile_blk * 4u + g) * 64u + rit;   // uint4 index
+        dst_qh[base * 4u + 0u] = qhw[g * 4u + 0u];
+        dst_qh[base * 4u + 1u] = qhw[g * 4u + 1u];
+        dst_qh[base * 4u + 2u] = qhw[g * 4u + 2u];
+        dst_qh[base * 4u + 3u] = qhw[g * 4u + 3u];
+    }
+
+    // scales: 16 chars contiguous per (row, block), tiled
+    __global char * s_dst = dst_s + (tile_blk * 64u + rit) * 16u;
+    #pragma unroll
+    for (int i = 0; i < 16; ++i) {
+        s_dst[i] = b->scales[i];
+    }
+
+    // super-block scale
+    dst_d[tile_blk * 64u + rit] = b->d;
+}
+
 kernel void kernel_convert_block_mxfp4_trans4_ns(
    global struct block_mxfp4 * src0,
    __global uint * dst_q,
@@ -4,6 +4,7 @@
 #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
 #define ADRENO_GPU 1
 #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
 #endif
 #define QK_K         256
 #define K_SCALE_SIZE 12
@@ -171,3 +172,319 @@ kernel void kernel_gemm_noshuffle_q4_k_f32(
        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
    }
 }
+
+// 1x8 per-WI tile (1 output row x 8 output cols). For the small-batch
+// (medium n_q, e.g. MTP/spec verify) path where the 2x8 kernel is starved:
+// at ne1<=8 the grid is (1, ceil(M/2)) -> only ~M/256 workgroups, leaving
+// the SP under-occupied. 1 row per WI doubles the M-axis workgroup count
+// (ceil(M/1)/128 vs ceil(M/2)/128) AND collapses the accumulators to a
+// single half8 (16 regs, no spill), so more waves co-reside. Same weight
+// traffic as 2x8 (rows never share weights); the win is pure occupancy.
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_128
+#endif
+kernel void kernel_gemm_noshuffle_q4_k_f32_r1(
+    global const ushort * src0_q,
+    global const uchar  * src0_s,
+    global const half   * src0_d,
+    global const half   * src0_dm,
+    read_only image1d_buffer_t src1,
+    global float * dst,
+    ulong offsetd,
+    int m,
+    int n,
+    int k,
+    int n_no_padding,
+    uchar mask_d6,
+    uchar mask_d4,
+    uchar mask_hi2
+) {
+    dst = (global float *)((global char *)dst + offsetd);
+    int n_4 = n >> 2;
+    int gy = get_global_id(0);
+    int gx = get_global_id(1);       // 1 row per WI
+
+    half8 c0 = 0;
+    half8 B;
+    half dq;
+
+    int num_blocks_K = k / QK_K;
+
+    global const ushort * weight_ptr = src0_q + gx;
+    global const half   * d_ptr      = src0_d  + gx;
+    global const half   * dm_ptr     = src0_dm + gx;
+
+    for (int i = 0; i < k; i += 32) {
+        int sb_idx  = i / QK_K;
+        int sub_idx = (i / 32) % 8;
+
+        half dd  = d_ptr [sb_idx * m];
+        half dmm = dm_ptr[sb_idx * m];
+
+        global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + gx;
+
+        uchar sv0, mn0;
+        get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+
+        half scale = convert_half(convert_float(dd)  * (float)sv0);
+        half mval  = convert_half(convert_float(dmm) * (float)mn0);
+
+        for (int l = 0; l < 32; l += 4) {
+            int ki = i + l;
+            ushort bits = weight_ptr[(ki/4) * m];
+
+            B.s0123 = read_imageh(src1, gy*2   + (ki+0) * n_4);
+            B.s4567 = read_imageh(src1, gy*2+1 + (ki+0) * n_4);
+            dq = (bits & 0x000F) * scale - mval;
+            c0 += B * dq;
+
+            B.s0123 = read_imageh(src1, gy*2   + (ki+1) * n_4);
+            B.s4567 = read_imageh(src1, gy*2+1 + (ki+1) * n_4);
+            dq = ((bits & 0x00F0) >> 4) * scale - mval;
+            c0 += B * dq;
+
+            B.s0123 = read_imageh(src1, gy*2   + (ki+2) * n_4);
+            B.s4567 = read_imageh(src1, gy*2+1 + (ki+2) * n_4);
+            dq = ((bits & 0x0F00) >> 8) * scale - mval;
+            c0 += B * dq;
+
+            B.s0123 = read_imageh(src1, gy*2   + (ki+3) * n_4);
+            B.s4567 = read_imageh(src1, gy*2+1 + (ki+3) * n_4);
+            dq = ((bits & 0xF000) >> 12) * scale - mval;
+            c0 += B * dq;
+        }
+    }
+
+    // Output: 8 cols, 1 row per col-step. Scalar store, coalesced across
+    // neighbouring WIs (consecutive gx -> consecutive dst addresses).
+    int idx = (gy<<3)*m + gx;
+    if (idx < m*n_no_padding) { dst[idx] = c0.s0; idx += m; }
+    if (idx < m*n_no_padding) { dst[idx] = c0.s1; idx += m; }
+    if (idx < m*n_no_padding) { dst[idx] = c0.s2; idx += m; }
+    if (idx < m*n_no_padding) { dst[idx] = c0.s3; idx += m; }
+    if (idx < m*n_no_padding) { dst[idx] = c0.s4; idx += m; }
+    if (idx < m*n_no_padding) { dst[idx] = c0.s5; idx += m; }
+    if (idx < m*n_no_padding) { dst[idx] = c0.s6; idx += m; }
+    if (idx < m*n_no_padding) { dst[idx] = c0.s7; }
+}
+
+// 2x8 tile, but weights read through an image1d_buffer (CL_R/UINT32 over the
+// same packed-q buffer) instead of a plain global buffer. The ne1==1 GEMV
+// already does this and is much faster per weight byte than this GEMM at
+// small n_q; the structural difference is the image path hits the dedicated
+// TPL1 weight cache (L1) while the global path only reaches L2. At small n_q
+// the forward is weight-read-bound, so L1-cached weights is the lever.
+// The 2 adjacent rows the 2x8 tile reads as a ushort2 are exactly one uint32,
+// so the vload2 becomes a single read_imageui at index gx + (ki/4)*(m/2).
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_128
+#endif
+kernel void kernel_gemm_noshuffle_q4_k_f32_kimg(
+    read_only image1d_buffer_t src0_q_img,
+    global const uchar  * src0_s,
+    global const half   * src0_d,
+    global const half   * src0_dm,
+    read_only image1d_buffer_t src1,
+    global float * dst,
+    ulong offsetd,
+    int m,
+    int n,
+    int k,
+    int n_no_padding,
+    uchar mask_d6,
+    uchar mask_d4,
+    uchar mask_hi2
+) {
+    dst = (global float *)((global char *)dst + offsetd);
+    int n_4 = n >> 2;
+    int m_2 = m >> 1;
+    int gy = get_global_id(0);
+    int gx = get_global_id(1);
+    int gx_2 = gx << 1;
+
+    half8 c0 = 0, c1 = 0;
+    half8 B;
+    half2 dequantized_weights;
+
+    int num_blocks_K = k / QK_K;
+
+    global const half * d_ptr  = src0_d  + gx_2;
+    global const half * dm_ptr = src0_dm + gx_2;
+
+    for (int i = 0; i < k; i += 32) {
+        int sb_idx  = i / QK_K;
+        int sub_idx = (i / 32) % 8;
+
+        half2 d  = vload2(0, d_ptr  + sb_idx * m);
+        half2 dm = vload2(0, dm_ptr + sb_idx * m);
+
+        global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + (gx_2+0);
+        global const uchar * sc1 = sc0 + 1;
+
+        uchar sv0, mn0, sv1, mn1;
+        get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+        get_scale_min_k4(sub_idx, sc1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+
+        half2 scale = convert_half2(convert_float2(d)  * convert_float2((uchar2)(sv0, sv1)));
+        half2 mval  = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));
+
+        for (int l = 0; l < 32; l += 4) {
+            int ki = i + l;
+            uint wpacked = read_imageui(src0_q_img, gx + (ki/4) * m_2).x;
+            ushort2 bits2 = (ushort2)((ushort)(wpacked & 0xFFFFu), (ushort)(wpacked >> 16));
+
+            // j=0
+            B.s0123 = read_imageh(src1, gy*2   + (ki+0) * n_4);
+            B.s4567 = read_imageh(src1, gy*2+1 + (ki+0) * n_4);
+            dequantized_weights.s0 = (bits2.s0 & 0x000F) * scale.s0 - mval.s0;
+            dequantized_weights.s1 = (bits2.s1 & 0x000F) * scale.s1 - mval.s1;
+            c0 += B * dequantized_weights.s0;
+            c1 += B * dequantized_weights.s1;
+
+            // j=1
+            B.s0123 = read_imageh(src1, gy*2   + (ki+1) * n_4);
+            B.s4567 = read_imageh(src1, gy*2+1 + (ki+1) * n_4);
+            dequantized_weights.s0 = ((bits2.s0 & 0x00F0) >> 4) * scale.s0 - mval.s0;
+            dequantized_weights.s1 = ((bits2.s1 & 0x00F0) >> 4) * scale.s1 - mval.s1;
+            c0 += B * dequantized_weights.s0;
+            c1 += B * dequantized_weights.s1;
+
+            // j=2
+            B.s0123 = read_imageh(src1, gy*2   + (ki+2) * n_4);
+            B.s4567 = read_imageh(src1, gy*2+1 + (ki+2) * n_4);
+            dequantized_weights.s0 = ((bits2.s0 & 0x0F00) >> 8) * scale.s0 - mval.s0;
+            dequantized_weights.s1 = ((bits2.s1 & 0x0F00) >> 8) * scale.s1 - mval.s1;
+            c0 += B * dequantized_weights.s0;
+            c1 += B * dequantized_weights.s1;
+
+            // j=3
+            B.s0123 = read_imageh(src1, gy*2   + (ki+3) * n_4);
+            B.s4567 = read_imageh(src1, gy*2+1 + (ki+3) * n_4);
+            dequantized_weights.s0 = ((bits2.s0 & 0xF000) >> 12) * scale.s0 - mval.s0;
+            dequantized_weights.s1 = ((bits2.s1 & 0xF000) >> 12) * scale.s1 - mval.s1;
+            c0 += B * dequantized_weights.s0;
+            c1 += B * dequantized_weights.s1;
+        }
+    }
+
+    int idx = (gy<<3)*m + (gx<<1);
+    if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s0, c1.s0), 0, dst + idx); idx += m; }
+    if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s1, c1.s1), 0, dst + idx); idx += m; }
+    if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s2, c1.s2), 0, dst + idx); idx += m; }
+    if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s3, c1.s3), 0, dst + idx); idx += m; }
+    if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s4, c1.s4), 0, dst + idx); idx += m; }
+    if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s5, c1.s5), 0, dst + idx); idx += m; }
+    if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s6, c1.s6), 0, dst + idx); idx += m; }
+    if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s7, c1.s7), 0, dst + idx); }
+}
+
+// Cooperative-K GEMM for the small-batch (n_q in [2..8]) path. Mirrors the
+// ne1==1 GEMV's structure: a WG is (COK_SG lanes x COK_NSG subgroups); each
+// lane owns ONE output row and computes its 8 (padded) columns, and the
+// COK_NSG subgroups SPLIT the K reduction round-robin, combining via a
+// __local reduction. This is the thing the per-WI GEMM lacked — at small n_q
+// the old kernel had ~M/256 workgroups each walking all of K serially; this
+// has M/64 workgroups AND COK_NSG-way K parallelism. Uses REQD_SUBGROUP_SIZE_64
+// + barrier (same safe reduction pattern as the GEMV; never sub_group_reduce
+// at full width on X2 per the GDN miscompile note).
+#define COK_NSG 8
+#define COK_SG  64
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemm_noshuffle_q4_k_f32_cok(
+    global const ushort * src0_q,
+    global const uchar  * src0_s,
+    global const half   * src0_d,
+    global const half   * src0_dm,
+    read_only image1d_buffer_t src1,
+    global float * dst,
+    ulong offsetd,
+    int m,
+    int n,
+    int k,
+    int n_no_padding,
+    uchar mask_d6,
+    uchar mask_d4,
+    uchar mask_hi2
+) {
+    dst = (global float *)((global char *)dst + offsetd);
+    int n_4  = n >> 2;
+    int gx   = get_global_id(0);     // output row
+    int sg   = get_local_id(1);      // subgroup index (K-split lane)
+    int lane = get_local_id(0);      // lane within subgroup (0..COK_SG-1)
+
+    int num_blocks_K = k / QK_K;
+    int num_32blk    = k / 32;
+
+    global const ushort * weight_ptr = src0_q + gx;
+    global const half   * d_ptr      = src0_d  + gx;
+    global const half   * dm_ptr     = src0_dm + gx;
+
+    half8 acc = 0;
+    half8 B;
+    half  dq;
+
+    for (int blk = sg; blk < num_32blk; blk += COK_NSG) {
+        int i       = blk << 5;       // blk * 32
+        int sb_idx  = blk >> 3;       // (blk*32) / QK_K  (QK_K = 256 = 32*8)
+        int sub_idx = blk & 7;        // (i/32) % 8
+
+        half dd  = d_ptr [sb_idx * m];
+        half dmm = dm_ptr[sb_idx * m];
+
+        global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + gx;
+        uchar sv0, mn0;
+        get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+        half scale = convert_half(convert_float(dd)  * (float)sv0);
+        half mval  = convert_half(convert_float(dmm) * (float)mn0);
+
+        for (int l = 0; l < 32; l += 4) {
+            int ki = i + l;
+            ushort bits = weight_ptr[(ki>>2) * m];
+
+            B.s0123 = read_imageh(src1,     (ki+0) * n_4);
+            B.s4567 = read_imageh(src1, 1 + (ki+0) * n_4);
+            dq = (bits & 0x000F) * scale - mval;
+            acc += B * dq;
+
+            B.s0123 = read_imageh(src1,     (ki+1) * n_4);
+            B.s4567 = read_imageh(src1, 1 + (ki+1) * n_4);
+            dq = ((bits & 0x00F0) >> 4) * scale - mval;
+            acc += B * dq;
+
+            B.s0123 = read_imageh(src1,     (ki+2) * n_4);
+            B.s4567 = read_imageh(src1, 1 + (ki+2) * n_4);
+            dq = ((bits & 0x0F00) >> 8) * scale - mval;
+            acc += B * dq;
+
+            B.s0123 = read_imageh(src1,     (ki+3) * n_4);
+            B.s4567 = read_imageh(src1, 1 + (ki+3) * n_4);
+            dq = ((bits & 0xF000) >> 12) * scale - mval;
+            acc += B * dq;
+        }
+    }
+
+    // cross-subgroup reduction over the K-split (float for accuracy)
+    local float8 reduceLM[COK_SG * (COK_NSG - 1)];
+    if (sg > 0) {
+        reduceLM[(sg - 1) * COK_SG + lane] = convert_float8(acc);
+    }
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (sg == 0) {
+        float8 sum = convert_float8(acc);
+        for (int s = 0; s < COK_NSG - 1; s++) {
+            sum += reduceLM[s * COK_SG + lane];
+        }
+        int idx = gx;
+        if (idx < m*n_no_padding) { dst[idx] = sum.s0; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s1; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s2; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s3; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s4; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s5; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s6; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s7; }
+    }
+}
@@ -5,6 +5,7 @@
 #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
 #define ADRENO_GPU 1
 #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
 #endif

 #ifdef ADRENO_GPU
@@ -138,3 +139,107 @@ kernel void kernel_gemm_noshuffle_q6_K_f32(
        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
    }
 }
+
+// Cooperative-K q6_K GEMM for the small-batch (n_q in [2..8]) path. Same idea
+// as the q4_K _cok kernel: WG = (COK_SG lanes x COK_NSG subgroups), each lane
+// owns ONE output row (half8 over the 8 padded cols), and the COK_NSG
+// subgroups split the K iterations round-robin and combine via a __local
+// reduction. Replaces the default 4-row-per-WI tile that walked all of K alone
+// (~M/512 WGs + serial reduction) at small n_q. REQD_SUBGROUP_SIZE_64 +
+// barrier (never sub_group_reduce at full width on X2).
+#define COK_NSG 8
+#define COK_SG  64
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemm_noshuffle_q6_K_f32_cok(
+        global const ushort * src0_ql,
+        global const uchar  * src0_qh,
+        global const ushort * src0_s,
+        global const half   * src0_d,
+        read_only image1d_buffer_t src1,
+        global float * dst,
+        ulong offsetd,
+        int m,
+        int n,
+        int k,
+        int n_no_padding,
+        ushort mask_f000,
+        uchar  mask_c0
+) {
+    dst = (global float *)( (global char *)dst + offsetd );
+
+    int n_4  = n >> 2;
+    int gx   = get_global_id(0);     // output row
+    int sg   = get_local_id(1);      // subgroup index (K-split)
+    int lane = get_local_id(0);      // lane within subgroup
+
+    global const ushort * ptr_ql = src0_ql + gx;
+    global const uchar  * ptr_qh = src0_qh + gx;
+    global const ushort * ptr_s  = src0_s  + gx;
+    global const half   * ptr_d  = src0_d  + gx;
+
+    half8 acc = 0;
+    half8 B;
+    half  dq;
+
+    int num_iter = k >> 2;   // k/4 iterations, 4 k-values each
+
+    for (int ib = sg; ib < num_iter; ib += COK_NSG) {
+        int i = ib << 2;     // ib * 4
+
+        ushort bits4 = ptr_ql[ib * m];          // ql for row gx at this 4-block
+        uchar  bits2 = ptr_qh[ib * m];          // qh
+
+        ushort s_packed = ptr_s[(i >> 5) * m];  // (i/16/2) = i/32
+        char2  sc2      = as_char2(s_packed);
+        char   scale_s  = (((i >> 4) & 1) == 0) ? sc2.s0 : sc2.s1; // (i/16)%2
+        half   scale_d  = ptr_d[(i >> 8) * m];  // i/256
+
+        // j=0
+        B.s0123 = read_imageh(src1, (i + 0)*n_4 + 0);
+        B.s4567 = read_imageh(src1, (i + 0)*n_4 + 1);
+        dq = (convert_half((bits4 & 0x000F) | ((bits2 & 0x03) << 4)) - 32.f) * scale_s * scale_d;
+        acc += B * dq;
+
+        // j=1
+        B.s0123 = read_imageh(src1, (i + 1)*n_4 + 0);
+        B.s4567 = read_imageh(src1, (i + 1)*n_4 + 1);
+        dq = (convert_half(((bits4 & 0x00F0) >> 4) | ((bits2 & 0x0C) << 2)) - 32.f) * scale_s * scale_d;
+        acc += B * dq;
+
+        // j=2
+        B.s0123 = read_imageh(src1, (i + 2)*n_4 + 0);
+        B.s4567 = read_imageh(src1, (i + 2)*n_4 + 1);
+        dq = (convert_half(((bits4 & 0x0F00) >> 8) | (bits2 & 0x30)) - 32.f) * scale_s * scale_d;
+        acc += B * dq;
+
+        // j=3
+        B.s0123 = read_imageh(src1, (i + 3)*n_4 + 0);
+        B.s4567 = read_imageh(src1, (i + 3)*n_4 + 1);
+        dq = (convert_half(((bits4 & mask_f000) >> 12) | ((bits2 & mask_c0) >> 2)) - 32.f) * scale_s * scale_d;
+        acc += B * dq;
+    }
+
+    local float8 reduceLM[COK_SG * (COK_NSG - 1)];
+    if (sg > 0) {
+        reduceLM[(sg - 1) * COK_SG + lane] = convert_float8(acc);
+    }
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (sg == 0) {
+        float8 sum = convert_float8(acc);
+        for (int s = 0; s < COK_NSG - 1; s++) {
+            sum += reduceLM[s * COK_SG + lane];
+        }
+        int idx = gx;
+        if (idx < m*n_no_padding) { dst[idx] = sum.s0; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s1; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s2; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s3; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s4; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s5; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s6; idx += m; }
+        if (idx < m*n_no_padding) { dst[idx] = sum.s7; }
+    }
+}
@@ -0,0 +1,136 @@
+// Batched (N>1) q6_K GEMM over the 64-row-TILED canonical layout produced by
+// kernel_convert_block_q6_k_tiled_ns (cvt.cl). Companion to the decode kernel
+// kernel_gemv_noshuffle_q6_K_f32_tiled: SAME pack, SAME canonical e-order
+// dequant (correct by construction vs reference ggml q6_K), extended to N output
+// columns. Makes the batched lm_head/embed (perplexity, spec-decode verify,
+// batched serving) correct on GPU while keeping the tiled convert the fast decode
+// GEMV depends on.
+//
+// One work-item owns one output ROW for a block of BN columns. A work-group is
+// {64 lanes, NTILES subgroups} = NTILES*64 rows; the global z dimension tiles the
+// N columns by BN. Each work-item computes its row's FULL K (no K-split, so no
+// cross-subgroup reduction), which lets the whole work-group share one staged
+// activation block:
+//
+//   __local activation staging — the BN columns of the current superblock (BN*256
+//   floats) are loaded into __local once per superblock, cooperatively by all
+//   NTILES*64 work-items, then every row reads its activation from __local. This
+//   removes the ~Nrows-fold redundant image reads of the first version (each lane
+//   re-read the activation), which made the batched GEMM ~2x slower than the plain
+//   noshuffle GEMM.
+//
+// Weights are read from __global (coalesced) — matching the decode kernel; the
+// lm_head weight is streamed with little reuse where coalesced global beats the
+// Adreno texture cache.
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define NTILES     4        // 64-row tiles per work-group (NTILES*64 = 256 rows)
+#define TILE_ROWS  64
+#define BN         16       // output columns handled per work-group (global z step)
+#define WG_THREADS (NTILES * TILE_ROWS)
+
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemm_noshuffle_q6_K_f32_tiled(
+    __global uint4 * src0_ql,   // tiled: 8 uint4 granules / superblock
+    __global uint4 * src0_qh,   // tiled: 4 uint4 granules / superblock
+    __global char  * src0_s,    // tiled: 16 chars / superblock
+    __global half  * src0_d,    // tiled: 1 half  / superblock
+    read_only image1d_buffer_t src1,   // activation [ne00, ne11] f32 (RGBA), column-major
+    global float * dst,
+    ulong offsetd,
+    int ne00,
+    int ne01,
+    int ne11
+) {
+    int rit = get_local_id(0);          // 0..63  (lane within a tile; coalesces weight loads)
+    int sg  = get_local_id(1);          // 0..NTILES-1
+    int lid = sg * TILE_ROWS + rit;     // 0..WG_THREADS-1 (flat local id)
+    int row = get_group_id(0) * WG_THREADS + lid;
+    int rt  = row / TILE_ROWS;          // global 64-row tile index
+    int col0 = get_global_id(2) * BN;   // first output column of this block
+
+    int nb = ne00 / 256;                // superblocks per row
+    int act_col_stride = ne00 / 4;      // activation float4 pixels per column
+
+    const bool row_ok = row < ne01;
+
+    // staged activation: BN columns x 256 elements for the current superblock
+    __local float lact[BN * 256];
+
+    float acc[BN];
+    #pragma unroll
+    for (int j = 0; j < BN; ++j) acc[j] = 0.0f;
+
+    for (int sb = 0; sb < nb; ++sb) {
+        // cooperatively stage BN columns' 256 activation elements (= BN*64 float4)
+        for (int p = lid; p < BN * 64; p += WG_THREADS) {
+            int j  = p >> 6;            // column within the BN block (p / 64)
+            int e4 = p & 63;            // element-quad within the column (p % 64)
+            int c  = col0 + j;
+            float4 v = (c < ne11)
+                ? read_imagef(src1, c * act_col_stride + sb * 64 + e4)
+                : (float4)(0.0f);
+            lact[p * 4 + 0] = v.x;
+            lact[p * 4 + 1] = v.y;
+            lact[p * 4 + 2] = v.z;
+            lact[p * 4 + 3] = v.w;      // lact[j*256 + e], e = e4*4 + t
+        }
+        barrier(CLK_LOCAL_MEM_FENCE);
+
+        if (row_ok) {
+            int tile_blk = rt * nb + sb;    // ne02 == 1 for lm_head/embed
+
+            float dval = (float)src0_d[tile_blk * TILE_ROWS + rit];
+            __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16;
+
+            uint ql[32];
+            uint qh[16];
+            #pragma unroll
+            for (int g = 0; g < 8; ++g) {
+                uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit];
+                ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w;
+            }
+            #pragma unroll
+            for (int g = 0; g < 4; ++g) {
+                uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit];
+                qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w;
+            }
+
+            // NOTE: the e loop (256) is deliberately NOT unrolled. Fully unrolling
+            // 256*BN MACs overflows the in-process Adreno compiler (host stack
+            // overflow at clBuildProgram, same class as the FA DK=512 OOM).
+            for (int e = 0; e < 256; ++e) {
+                uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF;
+                uint hi2  = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3;
+                int  code = (int)(low4 | (hi2 << 4)) - 32;
+                int  sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1);
+                float cs  = (float)code * (float)sc[sidx] * dval;
+                #pragma unroll
+                for (int j = 0; j < BN; ++j) {
+                    acc[j] += cs * lact[j * 256 + e];
+                }
+            }
+        }
+        barrier(CLK_LOCAL_MEM_FENCE);
+    }
+
+    if (row_ok) {
+        dst = (global float*)((global char*)dst + offsetd);
+        #pragma unroll
+        for (int j = 0; j < BN; ++j) {
+            int c = col0 + j;
+            if (c < ne11) {
+                dst[(ulong)c * ne01 + row] = acc[j];
+            }
+        }
+    }
+}
@@ -277,3 +277,107 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
    }

 }
+
+// Multi-column (N in [2..4]) variant of the q4_0 decode GEMV, for the speculative
+// / MTP verify batch (n_cols = 2..4 = drafted + bonus positions). Routes the small-
+// batch verify OFF the transposed-GEMM dead-zone (gemm_noshuffle_q4_0) onto the
+// efficient GEMV path. Each K-block's weights (regA hi+lo) are loaded ONCE and
+// reused across the n_cols activation columns. Per-column accumulation is
+// independent and identical to n_cols standalone GEMVs. n_cols==3 is byte-identical
+// to the original mc3 (col3 disabled, slots 6/7 stay zero). Kept the _mc3 name.
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+#define MC_DQ_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi
+#define MC_DQ_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo
+#else
+#define MC_DQ_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi
+#define MC_DQ_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo
+#endif
+// One column c: load this column's activation (own brace scope so the macros'
+// `shared_y` decl is re-scoped), then dequant (hi+lo) against the shared weights.
+#define MC_COL_Q40(ts, c) \
+    { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE     + slid*2 + k*8); \
+                      regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \
+      MC_DQ_HI(ts, as_ushort8(regA_hi), regS, regB); \
+      MC_DQ_LO(ts, as_ushort8(regA_lo), regS, regB); }
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+__kernel void kernel_gemv_noshuffle_q4_0_f32_mc3(
+        __read_only  image1d_buffer_t src0_q,  // quantized A
+        global half2  * src0_d,                // A scales
+        __read_only  image1d_buffer_t src1,    // B (n_cols columns, col-major image)
+        global float * dst,                    // C (column-major [M x n_cols])
+        ulong offsetd,
+        int ne00,                              // K
+        int ne01,                              // M
+        int n_cols)                            // N (2..4)
+{
+    uint groupId = get_local_id(1);
+    uint gid     = get_global_id(0);
+    ushort slid  = get_sub_group_local_id();
+
+    uint K = ne00;
+    uint M = ne01;
+
+    uint LINE_STRIDE_A  = M / 2;
+    // BLOCK_STRIDE_A is the LAYOUT stride between consecutive K-blocks = 4 uints
+    // per q4_0 block * M (set by the trans4_ns convert). The "4" is uints/block, NOT
+    // the subgroup count — keep it fixed so the K-split count (nsg) can vary.
+    uint BLOCK_STRIDE_A = N_SIMDGROUP * M;   // = 4 * M (N_SIMDGROUP is the #define 4)
+    uint COL_STRIDE     = K / 4;   // float4 pixels per activation column
+    uint nsg            = get_local_size(1);  // runtime K-split (4 default, 8 small-M)
+
+    __private uint4  regA_hi, regA_lo;
+    __private half2  regS;
+    __private float8 regB;
+
+    __private float2 ts0 = (float2)(0.0f);
+    __private float2 ts1 = (float2)(0.0f);
+    __private float2 ts2 = (float2)(0.0f);
+    __private float2 ts3 = (float2)(0.0f);
+
+    for (uint k = groupId; k < (K / QK4_0); k += nsg) {
+        regS = src0_d[gid + k * LINE_STRIDE_A];
+
+        // weights loaded ONCE, reused across the columns
+        regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+        MC_COL_Q40(ts0, 0);
+        MC_COL_Q40(ts1, 1);
+        if (n_cols > 2) MC_COL_Q40(ts2, 2);
+        if (n_cols > 3) MC_COL_Q40(ts3, 3);
+    }
+
+    // cross-subgroup reduce over nsg subgroups: pack the (up to 4) columns' float2
+    // into a float8. Generalized to runtime nsg (4 default, 8 for small-M). Each
+    // subgroup writes its partial; subgroup 0 sums the rest into its own acc. At
+    // nsg==4 this is byte-identical to the original (sums subgroups 1,2,3 in order).
+    __local float8 reduceLM[SIMDGROUP_WIDTH * 8];
+    float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1);
+    reduceLM[groupId * SIMDGROUP_WIDTH + slid] = acc;
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (groupId == 0) {
+        for (uint g = 1; g < nsg; g++) {
+            acc += reduceLM[g * SIMDGROUP_WIDTH + slid];
+        }
+        dst = (global float*)((global char*)dst + offsetd);
+        // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
+        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+        if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+        if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+    }
+}
+#undef MC_COL_Q40
+#undef MC_DQ_HI
+#undef MC_DQ_LO
@@ -286,3 +286,99 @@ kernel void kernel_gemv_noshuffle_q4_1_f32(
    }

 }
+
+// Multi-column (N in [2..4]) variant of the q4_1 decode GEMV (spec/MTP verify) =
+// q4_0 mc3 + the q4_1 per-block min (regM; dequant = q*scale + minv). n_cols=2..4;
+// routes the small-batch verify OFF the gemm_noshuffle_q4_1 dead-zone. n_cols==3 is
+// byte-identical to the original mc3. NB: this file spells the vec-broadcast define
+// BROADCAT (no S) — match it so the fast _8 path compiles.
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+#define MC_DQ1_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi
+#define MC_DQ1_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo
+#else
+#define MC_DQ1_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi
+#define MC_DQ1_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo
+#endif
+#define MC_COL_Q41(ts, c) \
+    { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE     + slid*2 + k*8); \
+                      regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \
+      MC_DQ1_HI(ts, as_ushort8(regA_hi), regS, regM, regB); \
+      MC_DQ1_LO(ts, as_ushort8(regA_lo), regS, regM, regB); }
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_1_f32_mc3(
+        read_only  image1d_buffer_t src0_q,
+        global half2  * src0_d,
+        global half2  * src0_m,
+        read_only  image1d_buffer_t src1,
+        global float * dst,
+        ulong offsetd,
+        int ne00,
+        int ne01,
+        int n_cols)
+{
+    uint groupId = get_local_id(1);
+    uint gid     = get_global_id(0);
+    ushort slid  = get_sub_group_local_id();
+
+    uint K = ne00;
+    uint M = ne01;
+
+    uint LINE_STRIDE_A  = M / 2;
+    uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+    uint COL_STRIDE     = K / 4;   // float4 pixels per activation column
+
+    private uint4  regA_hi, regA_lo;
+    private half2  regS, regM;
+    private float8 regB;
+
+    private float2 ts0 = (float2)(0.0f);
+    private float2 ts1 = (float2)(0.0f);
+    private float2 ts2 = (float2)(0.0f);
+    private float2 ts3 = (float2)(0.0f);
+
+    for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) {
+        regS = src0_d[gid + k * LINE_STRIDE_A];
+        regM = src0_m[gid + k * LINE_STRIDE_A];
+
+        // weights loaded ONCE, reused across the columns
+        regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+        MC_COL_Q41(ts0, 0);
+        MC_COL_Q41(ts1, 1);
+        if (n_cols > 2) MC_COL_Q41(ts2, 2);
+        if (n_cols > 3) MC_COL_Q41(ts3, 3);
+    }
+
+    // cross-subgroup reduce: pack the (up to 4) columns' float2 into a float8.
+    local float8 reduceLM[SUBGROUP_SIZE * 3];
+    float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1);
+    if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; }
+    if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; }
+    if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; }
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (groupId == 0) {
+        acc += reduceLM[SUBGROUP_SIZE * 0 + slid];
+        acc += reduceLM[SUBGROUP_SIZE * 1 + slid];
+        acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
+        dst = (global float*)((global char*)dst + offsetd);
+        // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
+        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+        if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+        if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+    }
+}
+#undef MC_COL_Q41
+#undef MC_DQ1_HI
+#undef MC_DQ1_LO
@@ -228,12 +228,20 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(
    uint groupId = get_local_id(1);
    uint gid     = get_global_id(0);
    ushort slid  = get_sub_group_local_id();
+    // K-split factor = #subgroups in the WG. Read from the launch (NOT a compile
+    // constant) so small-M projections (Kcur/Vcur/Qcur) can dispatch a wider
+    // K-split (more waves/SP -> latency hiding) while large-M keeps 4. The
+    // physical weight layout stride below is INDEPENDENT of this (see BLOCK_STRIDE_A).
+    uint nsg = get_local_size(1);

    uint K = ne00;
    uint M = ne01;

    uint LINE_STRIDE_A  = M / 2;
-    uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+    // Physical per-K-block stride in the packed image: 8 uints/block-row-pair *
+    // (M/2) row-pairs = 4*M uints. This is a layout constant, not tied to nsg.
+    uint BLOCK_STRIDE_A = 4 * M;
+    uint scales_per_row = (K / QK_K) * 12;

    // The x-grid is padded to CEIL_DIV(ne01/2,64)*64, so when ne01 % 128 != 0 the
    // tail lanes hold gid >= ne01/2. The output stores below are guarded, but the
@@ -259,7 +267,7 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(

    private float2 totalSum = (float2)(0.0f);

-    for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) {
+    for (uint k = groupId; k < (K / 32); k += nsg) {
        uint sb = k / 8;
        uint j  = k % 8;

@@ -303,28 +311,21 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(
 #endif // VECTOR_SUB_GROUP_BROADCAST
    }

-    // reduction in local memory, assumes #wave=4
-    local float2 reduceLM[SUBGROUP_SIZE * 3];
-    if (groupId == 1) {
-        reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum;
-    }
-    if (groupId == 2) {
-        reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum;
-    }
-    if (groupId == 3) {
-        reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum;
+    // Cross-subgroup reduction in local memory. Generalized to nsg subgroups
+    // (was a hard-coded 4-wave unroll). Sized for up to 16 subgroups (the widest
+    // K-split we dispatch for small M). At nsg==4 the accumulation order is
+    // identical to the original unroll -> byte-identical for the large-M path.
+    local float2 reduceLM[SUBGROUP_SIZE * 15];
+    if (groupId > 0) {
+        reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = totalSum;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if (groupId == 0) {
-        totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid];
-    }
-    if (groupId == 0) {
-        totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid];
-    }
-    if (groupId == 0) {
-        totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid];
+        for (uint i = 0; i < nsg - 1; ++i) {
+            totalSum += reduceLM[SUBGROUP_SIZE * i + slid];
+        }
    }

    // 2 outputs per fiber in wave 0
@@ -339,3 +340,484 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(
    }

 }
+
+// --- Fused gate+up GEMV + GLU epilogue (FFN) ------------------------------------
+// Folds the FFN's two decode GEMVs (ffn_gate, ffn_up) and the following GLU into a
+// SINGLE dispatch: {MUL_MAT(Wg,x), MUL_MAT(Wu,x), GLU}. Both matmuls share the same
+// activation x (ffn_norm), so the activation image read is issued ONCE per K-block
+// and reused for the gate and up dot products (the per-op path re-reads it twice and
+// also materializes the two full ffn-wide intermediates to global, which the GLU
+// then re-reads). The gate/up partial sums are accumulated in the SAME per-fiber
+// order and reduced in the SAME cross-subgroup order as the standalone GEMV, and the
+// GLU formula is the exact scalar expression from kernels/glu.cl, so the output is
+// BYTE-IDENTICAL to the per-op matmul+matmul+glu path -> safe to default on.
+//   glu_op: REGLU=0, GEGLU=1, SWIGLU=2, GEGLU_ERF=4, GEGLU_QUICK=5 (ggml_glu_op).
+// Weights: src0g_* = gate (= GLU src[0]); src0u_* = up (= GLU src[1]).
+#define GLU_GEGLU_COEF_A      0.044715f
+#define GLU_SQRT_2_OVER_PI    0.79788456080286535587989211986876f
+#define GLU_SQRT_2_INV        0.70710678118654752440084436210484f
+#define GLU_QUICK_COEF       -1.702f
+
+inline float glu_apply(int glu_op, float g, float u) {
+    float act;
+    if (glu_op == 1) {        // GEGLU (tanh-approx gelu)
+        act = 0.5f*g*(1.0f + tanh(GLU_SQRT_2_OVER_PI*g*(1.0f + GLU_GEGLU_COEF_A*g*g)));
+    } else if (glu_op == 2) { // SWIGLU (silu)
+        act = g / (1.0f + exp(-g));
+    } else if (glu_op == 0) { // REGLU
+        return g*u*(g > 0.0f);
+    } else if (glu_op == 4) { // GEGLU_ERF
+        act = 0.5f*g*(1.0f + erf(g*GLU_SQRT_2_INV));
+    } else {                  // GEGLU_QUICK (glu_op == 5)
+        act = g*(1.0f/(1.0f + exp(GLU_QUICK_COEF*g)));
+    }
+    return act*u;
+}
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_glu(
+        read_only  image1d_buffer_t src0g_q,
+        global half2  * src0g_d,
+        global half2  * src0g_m,
+        global uchar  * src0g_s,
+        read_only  image1d_buffer_t src0u_q,
+        global half2  * src0u_d,
+        global half2  * src0u_m,
+        global uchar  * src0u_s,
+        read_only  image1d_buffer_t src1,
+        global float * dst,
+        ulong offsetd,
+        int ne00,
+        int ne01,
+        int glu_op,
+        uchar mask_d6,
+        uchar mask_d4,
+        uchar mask_hi2)
+{
+    uint groupId = get_local_id(1);
+    uint gid     = get_global_id(0);
+    ushort slid  = get_sub_group_local_id();
+    uint nsg     = get_local_size(1);
+
+    uint K = ne00;
+    uint M = ne01;
+
+    uint LINE_STRIDE_A  = M / 2;
+    uint BLOCK_STRIDE_A = 4 * M;
+
+    private uint4  regA;
+    private half2  regS, regM;
+    private float8 regB;
+
+    private float2 gateSum = (float2)(0.0f);
+    private float2 upSum   = (float2)(0.0f);
+
+    // Two SEQUENTIAL K-loops (gate fully, then up). Keeping only one weight's
+    // working set live at a time holds the kernel's register footprint at ~the
+    // base single-weight GEMV's, so its max WG stays 1024 (16 subgroups) and the
+    // per-subgroup K-split matches the standalone wide GEMV exactly -> the gate
+    // and up partial sums are BYTE-IDENTICAL to the per-op path. The macro body
+    // is the base kernel's inner loop verbatim, parameterized by weight source.
+#define Q4K_GLU_LOOP(SUM, Q, DD, MM, SS)                                                       \
+    for (uint k = groupId; k < (K / 32); k += nsg) {                                           \
+        uint sb = k / 8;                                                                       \
+        uint j  = k % 8;                                                                       \
+        half2 d   = DD[gid + sb * LINE_STRIDE_A];                                              \
+        half2 dm  = MM[gid + sb * LINE_STRIDE_A];                                              \
+        global const uchar * sc0 = SS + sb * 12 * M + 2 * gid;                                 \
+        global const uchar * sc1 = sc0 + 1;                                                    \
+        uchar sv0, mn0, sv1, mn1;                                                              \
+        get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);                      \
+        get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);                      \
+        regS = convert_half2(convert_float2(d)  * convert_float2((uchar2)(sv0, sv1)));         \
+        regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));         \
+        if (slid < 4) {                                                                        \
+            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));                                \
+            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));                            \
+        }                                                                                      \
+        regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;           \
+        regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;           \
+        regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;           \
+        regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;           \
+        DEQ_HI(SUM, as_ushort8(regA), regS, regM, regB);                                       \
+        regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;           \
+        regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;           \
+        regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;           \
+        regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;           \
+        DEQ_LO(SUM, as_ushort8(regA), regS, regM, regB);                                       \
+    }
+
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+#define DEQ_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi
+#define DEQ_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo
+#else
+#define DEQ_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi
+#define DEQ_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo
+#endif
+
+    Q4K_GLU_LOOP(gateSum, src0g_q, src0g_d, src0g_m, src0g_s)
+    Q4K_GLU_LOOP(upSum,   src0u_q, src0u_d, src0u_m, src0u_s)
+
+#undef DEQ_HI
+#undef DEQ_LO
+#undef Q4K_GLU_LOOP
+
+    // Cross-subgroup reduction in local memory. Packs gate (xy) + up (zw) into a
+    // float4 so both reduce in one pass; summation order matches the base GEMV's
+    // per-channel loop -> byte-identical partial sums.
+    local float4 reduceLM[SUBGROUP_SIZE * 15];
+    if (groupId > 0) {
+        reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = (float4)(gateSum, upSum);
+    }
+    barrier(CLK_LOCAL_MEM_FENCE);
+    if (groupId == 0) {
+        for (uint i = 0; i < nsg - 1; ++i) {
+            float4 p = reduceLM[SUBGROUP_SIZE * i + slid];
+            gateSum += p.xy;
+            upSum   += p.zw;
+        }
+        dst = (global float*)((global char*)dst + offsetd);
+        dst[gid * 2 + 0] = glu_apply(glu_op, gateSum.s0, upSum.s0);
+        dst[gid * 2 + 1] = glu_apply(glu_op, gateSum.s1, upSum.s1);
+    }
+}
+
+// --- Split-K-across-workgroups decode GEMV (small-M projections) ----------------
+// A single-token GEMV makes only ceil(M/2/64) workgroups; a WG runs on one Adreno
+// compute unit, so for small M (Kcur/Vcur, M=512 -> 4 WGs) most of the 16 CUs sit
+// idle and the matmul is bandwidth-starved even with a wide intra-WG K-split. This
+// variant adds a SECOND grid dimension of `ksplit` workgroups that each reduce a
+// disjoint slice of K and write a per-slice partial; kernel_gemv_splitk_reduce_f32
+// then sums the partials into dst. Identical math/layout to the base kernel
+// (physical block stride 4*M, get_scale_min_k4) -> coherent. Gated host-side to
+// M<=1024 (M>=2048
+// already fills the CUs and the extra reduce dispatch only hurts).
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_splitk(
+        read_only  image1d_buffer_t src0_q,
+        global half2  * src0_d,
+        global half2  * src0_m,
+        global uchar  * src0_s,
+        read_only  image1d_buffer_t src1,
+        global float * partial,          // [ksplit * M], slice-major
+        int ne00,
+        int ne01,
+        uchar mask_d6,
+        uchar mask_d4,
+        uchar mask_hi2)
+{
+    uint groupId = get_local_id(1);
+    uint gid     = get_global_id(0);
+    ushort slid  = get_sub_group_local_id();
+    uint nsg     = get_local_size(1);
+    uint ksplit  = get_num_groups(1);
+    uint kslice  = get_group_id(1);
+
+    uint K = ne00;
+    uint M = ne01;
+    uint LINE_STRIDE_A  = M / 2;
+    uint BLOCK_STRIDE_A = 4 * M;      // physical, independent of the K-split
+
+    private uint4  regA;
+    private half2  regS, regM;
+    private float8 regB;
+    private float2 totalSum = (float2)(0.0f);
+
+    // each (kslice, subgroup) pair owns a disjoint set of K-blocks
+    for (uint k = kslice * nsg + groupId; k < (K / 32); k += ksplit * nsg) {
+        uint sb = k / 8;
+        uint j  = k % 8;
+        half2 d   = src0_d[gid + sb * LINE_STRIDE_A];
+        half2 dm  = src0_m[gid + sb * LINE_STRIDE_A];
+        global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid;
+        global const uchar * sc1 = sc0 + 1;
+        uchar sv0, mn0, sv1, mn1;
+        get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+        get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+        regS = convert_half2(convert_float2(d)  * convert_float2((uchar2)(sv0, sv1)));
+        regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));
+        if (slid < 4) {
+            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
+            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
+        }
+        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+        dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB);
+#else
+        dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB);
+#endif
+        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+        dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB);
+#else
+        dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regM, regB);
+#endif
+    }
+
+    local float2 reduceLM[SUBGROUP_SIZE * 15];
+    if (groupId > 0) {
+        reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = totalSum;
+    }
+    barrier(CLK_LOCAL_MEM_FENCE);
+    if (groupId == 0) {
+        for (uint i = 0; i < nsg - 1; ++i) {
+            totalSum += reduceLM[SUBGROUP_SIZE * i + slid];
+        }
+        vstore2(totalSum, 0, &(partial[kslice * M + gid * 2]));
+    }
+}
+
+// Sum the per-slice partials [ksplit * M] into dst[M]; applies the dst byte offset.
+kernel void kernel_gemv_splitk_reduce_f32(
+        global float * partial,
+        global float * dst,
+        ulong offsetd,
+        int   ne01,         // M
+        int   ksplit)
+{
+    uint r = get_global_id(0);
+    if (r >= (uint)ne01) return;
+    float acc = 0.0f;
+    for (uint s = 0; s < (uint)ksplit; ++s) {
+        acc += partial[s * (uint)ne01 + r];
+    }
+    dst = (global float*)((global char*)dst + offsetd);
+    dst[r] = acc;
+}
+
+
+// --- Dequant-once macros for the mc3 verify GEMV (Q4K_MC3_DEQUANT_ONCE) ---
+// The inline dequantizeBlockAccum_* macros recompute the dequantized weight
+// ((code & mask)>>shift)*scale - minv ONCE PER COLUMN (3x), and the flat
+// 32-FMA unroll spills ~430 B of temporaries. These macros split the work:
+// DEQUANT_Q4K_BLOCK computes the 16 weights/row of one 32-block ONCE into a
+// half2[] (row0 in .s0, row1 in .s1) — stored as half, the exact type the
+// inline expression yields (int*half-half), so no extra rounding. MAC_Q4K_BLOCK
+// then accumulates them against a column's broadcast activation in the SAME
+// per-accumulator order as the inline macro. Each weight value and each
+// accumulator's add-chain is bit-for-bit identical => byte-identical output,
+// while the dequant ALU drops 3x->1x and the live set shrinks. Requires the
+// Qualcomm vector sub_group_broadcast (float8); enabled opt-in on Adreno.
+#define DEQ_Q4K_HALF2(b0, b1, msk, sh, scale, minv) \
+    (half2)( ((b0 & msk) >> sh) * scale.s0 - minv.s0, \
+             ((b1 & msk) >> sh) * scale.s1 - minv.s1 )
+
+#define DEQUANT_Q4K_BLOCK(wq, bits, scale, minv) \
+    wq[0]  = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x000F, 0,  scale, minv); \
+    wq[1]  = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x00F0, 4,  scale, minv); \
+    wq[2]  = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x0F00, 8,  scale, minv); \
+    wq[3]  = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0xF000, 12, scale, minv); \
+    wq[4]  = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x000F, 0,  scale, minv); \
+    wq[5]  = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x00F0, 4,  scale, minv); \
+    wq[6]  = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x0F00, 8,  scale, minv); \
+    wq[7]  = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0xF000, 12, scale, minv); \
+    wq[8]  = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x000F, 0,  scale, minv); \
+    wq[9]  = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x00F0, 4,  scale, minv); \
+    wq[10] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x0F00, 8,  scale, minv); \
+    wq[11] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0xF000, 12, scale, minv); \
+    wq[12] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x000F, 0,  scale, minv); \
+    wq[13] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x00F0, 4,  scale, minv); \
+    wq[14] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x0F00, 8,  scale, minv); \
+    wq[15] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0xF000, 12, scale, minv);
+
+// ln0/ln1 = the two source lanes whose activation float8 this block consumes
+// (0,1 for the hi block, 2,3 for the lo block — matching the inline _hi/_lo).
+#define MAC_Q4K_BLOCK(ts, wq, y, ln0, ln1) { \
+    float8 sy = sub_group_broadcast(y, ln0); \
+    ts.s0 += wq[0].s0*sy.s0; ts.s0 += wq[1].s0*sy.s1; ts.s0 += wq[2].s0*sy.s2; ts.s0 += wq[3].s0*sy.s3; \
+    ts.s0 += wq[4].s0*sy.s4; ts.s0 += wq[5].s0*sy.s5; ts.s0 += wq[6].s0*sy.s6; ts.s0 += wq[7].s0*sy.s7; \
+    ts.s1 += wq[0].s1*sy.s0; ts.s1 += wq[1].s1*sy.s1; ts.s1 += wq[2].s1*sy.s2; ts.s1 += wq[3].s1*sy.s3; \
+    ts.s1 += wq[4].s1*sy.s4; ts.s1 += wq[5].s1*sy.s5; ts.s1 += wq[6].s1*sy.s6; ts.s1 += wq[7].s1*sy.s7; \
+    sy = sub_group_broadcast(y, ln1); \
+    ts.s0 += wq[8].s0*sy.s0;  ts.s0 += wq[9].s0*sy.s1;  ts.s0 += wq[10].s0*sy.s2; ts.s0 += wq[11].s0*sy.s3; \
+    ts.s0 += wq[12].s0*sy.s4; ts.s0 += wq[13].s0*sy.s5; ts.s0 += wq[14].s0*sy.s6; ts.s0 += wq[15].s0*sy.s7; \
+    ts.s1 += wq[8].s1*sy.s0;  ts.s1 += wq[9].s1*sy.s1;  ts.s1 += wq[10].s1*sy.s2; ts.s1 += wq[11].s1*sy.s3; \
+    ts.s1 += wq[12].s1*sy.s4; ts.s1 += wq[13].s1*sy.s5; ts.s1 += wq[14].s1*sy.s6; ts.s1 += wq[15].s1*sy.s7; \
+}
+
+// Multi-column (N=3) variant of the q4_K decode GEMV, for the speculative /
+// MTP verify batch (ne1=3 = 2 drafts + 1 bonus). Stays on the efficient GEMV
+// path (subgroup-broadcast activation, NSUBGROUPS K-split) instead of the
+// transposed-GEMM dead-zone path. Each K-block's weights (regA_hi/regA_lo) are
+// loaded ONCE and reused across all 3 activation columns — same weight traffic
+// as one decode, ~3x the (cheap) dequant ALU. Per-column accumulation is
+// independent and identical to 3 standalone GEMVs => byte-identical, so it does
+// NOT perturb the lm_head logits / spec accept rate.
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_mc3(
+        read_only  image1d_buffer_t src0_q,
+        global half2  * src0_d,
+        global half2  * src0_m,
+        global uchar  * src0_s,
+        read_only  image1d_buffer_t src1,
+        global float * dst,
+        ulong offsetd,
+        int ne00,
+        int ne01,
+        uchar mask_d6,
+        uchar mask_d4,
+        uchar mask_hi2)
+{
+    uint groupId = get_local_id(1);
+    uint gid     = get_global_id(0);
+    ushort slid  = get_sub_group_local_id();
+
+    uint K = ne00;
+    uint M = ne01;
+
+    uint LINE_STRIDE_A  = M / 2;
+    uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+    uint COL_STRIDE     = K / 4;   // float4 pixels per activation column
+
+    private uint4  regA_hi, regA_lo;
+    private half2  regS, regM;
+    private float8 regB;
+
+    private float2 ts0 = (float2)(0.0f);
+    private float2 ts1 = (float2)(0.0f);
+    private float2 ts2 = (float2)(0.0f);
+
+#ifdef Q4K_MC3_DEQUANT_LDS
+    // One 16-half2 block buffer per WI (reused hi->lo): forces the dequantized
+    // weights into LDS instead of private arrays (which spill to slow global on
+    // Adreno). 64*NSUBGROUPS WIs * 16 half2 = 16 KB; each WI owns its own slot
+    // range (flat*16) -> no cross-lane sharing, no barrier needed.
+    local half2 wstage[SUBGROUP_SIZE * NSUBGROUPS * 16];
+    local half2 * ws = wstage + (groupId * SUBGROUP_SIZE + slid) * 16;
+#endif
+
+    for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) {
+        uint sb = k / 8;
+        uint j  = k % 8;
+
+        half2 d   = src0_d[gid + sb * LINE_STRIDE_A];
+        half2 dm  = src0_m[gid + sb * LINE_STRIDE_A];
+
+        global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid;
+        global const uchar * sc1 = sc0 + 1;
+
+        uchar sv0, mn0, sv1, mn1;
+        get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+        get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+
+        regS = convert_half2(convert_float2(d)  * convert_float2((uchar2)(sv0, sv1)));
+        regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));
+
+        // weights loaded ONCE, reused across the 3 columns
+        regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+#ifdef Q4K_MC3_DEQUANT_ONCE
+        // Dequant the 32 weights/row (16 hi + 16 lo) ONCE into half2[] (byte-
+        // identical to the inline intermediate), then MAC against each column's
+        // activation. Drops the dequant ALU 3x->1x and the macro-temp spill.
+        half2 wq_hi[16], wq_lo[16];
+        DEQUANT_Q4K_BLOCK(wq_hi, as_ushort8(regA_hi), regS, regM);
+        DEQUANT_Q4K_BLOCK(wq_lo, as_ushort8(regA_lo), regS, regM);
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts0, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts0, wq_lo, regB, 2, 3); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts1, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts1, wq_lo, regB, 2, 3); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts2, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts2, wq_lo, regB, 2, 3); }
+#elif defined(Q4K_MC3_DEQUANT_LDS)
+        // LDS-staged dequant: dequant a 32-block ONCE into the per-WI LDS slot
+        // (hi pass then lo pass, overwriting), MAC each column from LDS. ts*
+        // receive hi-then-lo in the same order as DEQUANT_ONCE -> byte-identical.
+        // Activations reloaded per pass (cheap, imaged); only one regB + 0 weight
+        // regs live -> the weight working set lives in LDS, not spilled private.
+        DEQUANT_Q4K_BLOCK(ws, as_ushort8(regA_hi), regS, regM);
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts0, ws, regB, 0, 1); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts1, ws, regB, 0, 1); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts2, ws, regB, 0, 1); }
+        DEQUANT_Q4K_BLOCK(ws, as_ushort8(regA_lo), regS, regM);
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts0, ws, regB, 2, 3); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts1, ws, regB, 2, 3); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+          MAC_Q4K_BLOCK(ts2, ws, regB, 2, 3); }
+#else
+        // Per-column: load only this column's activation (single regB live at a
+        // time -> 1/3 the activation register pressure vs holding all 3) then
+        // dequant against the shared weights. Cuts the private-mem spill.
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts0, as_ushort8(regA_hi), regS, regM, regB);
+          dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts0, as_ushort8(regA_lo), regS, regM, regB); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts1, as_ushort8(regA_hi), regS, regM, regB);
+          dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts1, as_ushort8(regA_lo), regS, regM, regB); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts2, as_ushort8(regA_hi), regS, regM, regB);
+          dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts2, as_ushort8(regA_lo), regS, regM, regB); }
+#else
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts0, as_ushort8(regA_hi), regS, regM, regB);
+          dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts0, as_ushort8(regA_lo), regS, regM, regB); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts1, as_ushort8(regA_hi), regS, regM, regB);
+          dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts1, as_ushort8(regA_lo), regS, regM, regB); }
+        { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE     + slid*2 + k*8);
+                          regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts2, as_ushort8(regA_hi), regS, regM, regB);
+          dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts2, as_ushort8(regA_lo), regS, regM, regB); }
+#endif
+#endif // Q4K_MC3_DEQUANT_ONCE
+    }
+
+    // cross-subgroup reduce: pack the 3 columns' float2 into a float8 (6 used).
+    local float8 reduceLM[SUBGROUP_SIZE * 3];
+    float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, 0.0f, 0.0f);
+    if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; }
+    if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; }
+    if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; }
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (groupId == 0) {
+        acc += reduceLM[SUBGROUP_SIZE * 0 + slid];
+        acc += reduceLM[SUBGROUP_SIZE * 1 + slid];
+        acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
+        dst = (global float*)((global char*)dst + offsetd);
+        // dst is column-major [M rows x 3 cols]: (row, col) at col*M + row
+        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+        vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+    }
+}
@@ -0,0 +1,349 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define QK_K  256
+#define NSUBGROUPS 4
+#define SUBGROUP_SIZE 64
+
+// scales are transposed: consecutive codes of a row are `stride` apart
+inline void get_scale_min_k4(
+    int j,
+    global const uchar * q,
+    uint stride,
+    uchar * d,
+    uchar * m,
+    uchar mask_d6,
+    uchar mask_d4,
+    uchar mask_hi2
+) {
+    if (j < 4) {
+        *d = q[j*stride]     & mask_d6;
+        *m = q[(j+4)*stride] & mask_d6;
+    } else {
+        *d = (q[(j+4)*stride] & mask_d4) | ((q[(j-4)*stride] & mask_hi2) >> 2);
+        *m = ((q[(j+4)*stride] >> 4) & mask_d4) | ((q[j*stride] & mask_hi2) >> 2);
+    }
+}
+
+#define dequantizeBlockAccum_ns_sgbroadcast_1_hi(total_sums, bits4, scale, minv, y) \
+    float shared_y; \
+    shared_y = sub_group_broadcast(y.s0, 0); \
+    total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s1, 0); \
+    total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s2, 0); \
+    total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s3, 0); \
+    total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s4, 0); \
+    total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s5, 0); \
+    total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s6, 0); \
+    total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s7, 0); \
+    total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s0, 1); \
+    total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s1, 1); \
+    total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s2, 1); \
+    total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s3, 1); \
+    total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s4, 1); \
+    total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s5, 1); \
+    total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s6, 1); \
+    total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s7, 1); \
+    total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+
+
+#define dequantizeBlockAccum_ns_sgbroadcast_1_lo(total_sums, bits4, scale, minv, y) \
+    shared_y = sub_group_broadcast(y.s0, 2); \
+    total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s1, 2); \
+    total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s2, 2); \
+    total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s3, 2); \
+    total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s4, 2); \
+    total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s5, 2); \
+    total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s6, 2); \
+    total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s7, 2); \
+    total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s0, 3); \
+    total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s1, 3); \
+    total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s2, 3); \
+    total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s3, 3); \
+    total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s4, 3); \
+    total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s5, 3); \
+    total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s6, 3); \
+    total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+    shared_y = sub_group_broadcast(y.s7, 3); \
+    total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+    total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+
+
+#define dequantizeBlockAccum_ns_sgbroadcast_8_hi(total_sums, bits4, scale, minv, y) \
+    float8 shared_y; \
+    shared_y = sub_group_broadcast(y, 0); \
+    total_sums.s0 += ((bits4.s0 & 0x000F)         * scale.s0 - minv.s0) * shared_y.s0; \
+    total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4)  * scale.s0 - minv.s0) * shared_y.s1; \
+    total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8)  * scale.s0 - minv.s0) * shared_y.s2; \
+    total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \
+    total_sums.s0 += ((bits4.s2 & 0x000F)         * scale.s0 - minv.s0) * shared_y.s4; \
+    total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4)  * scale.s0 - minv.s0) * shared_y.s5; \
+    total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8)  * scale.s0 - minv.s0) * shared_y.s6; \
+    total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \
+    total_sums.s1 += ((bits4.s1 & 0x000F)         * scale.s1 - minv.s1) * shared_y.s0; \
+    total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4)  * scale.s1 - minv.s1) * shared_y.s1; \
+    total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8)  * scale.s1 - minv.s1) * shared_y.s2; \
+    total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \
+    total_sums.s1 += ((bits4.s3 & 0x000F)         * scale.s1 - minv.s1) * shared_y.s4; \
+    total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4)  * scale.s1 - minv.s1) * shared_y.s5; \
+    total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8)  * scale.s1 - minv.s1) * shared_y.s6; \
+    total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \
+    shared_y = sub_group_broadcast(y, 1); \
+    total_sums.s0 += ((bits4.s4 & 0x000F)         * scale.s0 - minv.s0) * shared_y.s0; \
+    total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4)  * scale.s0 - minv.s0) * shared_y.s1; \
+    total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8)  * scale.s0 - minv.s0) * shared_y.s2; \
+    total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \
+    total_sums.s0 += ((bits4.s6 & 0x000F)         * scale.s0 - minv.s0) * shared_y.s4; \
+    total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4)  * scale.s0 - minv.s0) * shared_y.s5; \
+    total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8)  * scale.s0 - minv.s0) * shared_y.s6; \
+    total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \
+    total_sums.s1 += ((bits4.s5 & 0x000F)         * scale.s1 - minv.s1) * shared_y.s0; \
+    total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4)  * scale.s1 - minv.s1) * shared_y.s1; \
+    total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8)  * scale.s1 - minv.s1) * shared_y.s2; \
+    total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \
+    total_sums.s1 += ((bits4.s7 & 0x000F)         * scale.s1 - minv.s1) * shared_y.s4; \
+    total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4)  * scale.s1 - minv.s1) * shared_y.s5; \
+    total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8)  * scale.s1 - minv.s1) * shared_y.s6; \
+    total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \
+
+
+#define dequantizeBlockAccum_ns_sgbroadcast_8_lo(total_sums, bits4, scale, minv, y) \
+    shared_y = sub_group_broadcast(y, 2); \
+    total_sums.s0 += ((bits4.s0 & 0x000F)         * scale.s0 - minv.s0) * shared_y.s0; \
+    total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4)  * scale.s0 - minv.s0) * shared_y.s1; \
+    total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8)  * scale.s0 - minv.s0) * shared_y.s2; \
+    total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \
+    total_sums.s0 += ((bits4.s2 & 0x000F)         * scale.s0 - minv.s0) * shared_y.s4; \
+    total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4)  * scale.s0 - minv.s0) * shared_y.s5; \
+    total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8)  * scale.s0 - minv.s0) * shared_y.s6; \
+    total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \
+    total_sums.s1 += ((bits4.s1 & 0x000F)         * scale.s1 - minv.s1) * shared_y.s0; \
+    total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4)  * scale.s1 - minv.s1) * shared_y.s1; \
+    total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8)  * scale.s1 - minv.s1) * shared_y.s2; \
+    total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \
+    total_sums.s1 += ((bits4.s3 & 0x000F)         * scale.s1 - minv.s1) * shared_y.s4; \
+    total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4)  * scale.s1 - minv.s1) * shared_y.s5; \
+    total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8)  * scale.s1 - minv.s1) * shared_y.s6; \
+    total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \
+    shared_y = sub_group_broadcast(y, 3); \
+    total_sums.s0 += ((bits4.s4 & 0x000F)         * scale.s0 - minv.s0) * shared_y.s0; \
+    total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4)  * scale.s0 - minv.s0) * shared_y.s1; \
+    total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8)  * scale.s0 - minv.s0) * shared_y.s2; \
+    total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \
+    total_sums.s0 += ((bits4.s6 & 0x000F)         * scale.s0 - minv.s0) * shared_y.s4; \
+    total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4)  * scale.s0 - minv.s0) * shared_y.s5; \
+    total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8)  * scale.s0 - minv.s0) * shared_y.s6; \
+    total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \
+    total_sums.s1 += ((bits4.s5 & 0x000F)         * scale.s1 - minv.s1) * shared_y.s0; \
+    total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4)  * scale.s1 - minv.s1) * shared_y.s1; \
+    total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8)  * scale.s1 - minv.s1) * shared_y.s2; \
+    total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \
+    total_sums.s1 += ((bits4.s7 & 0x000F)         * scale.s1 - minv.s1) * shared_y.s4; \
+    total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4)  * scale.s1 - minv.s1) * shared_y.s5; \
+    total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8)  * scale.s1 - minv.s1) * shared_y.s6; \
+    total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_o4(
+        read_only  image1d_buffer_t src0_q,
+        global half2  * src0_d,
+        global half2  * src0_m,
+        global uchar  * src0_s,
+        read_only  image1d_buffer_t src1,
+        global float * dst,
+        ulong offsetd,
+        int ne00,
+        int ne01,
+        uchar mask_d6,
+        uchar mask_d4,
+        uchar mask_hi2)
+{
+    uint groupId = get_local_id(1);
+    uint gid     = get_global_id(0);        // 4-output quad index
+    ushort slid  = get_sub_group_local_id();
+
+    // Two consecutive pair-indices (each the same access pattern the 2-output
+    // kernel uses); together they cover 4 consecutive output rows.
+    uint gid_a = gid * 2;
+    uint gid_b = gid * 2 + 1;
+
+    uint K = ne00;
+    uint M = ne01;
+
+    uint LINE_STRIDE_A  = M / 2;
+    uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+
+    private uint4  regA;
+    private half2  regS_a, regS_b;
+    private half2  regM_a, regM_b;
+    private float8 regB;
+
+    private float2 totalSum_a = (float2)(0.0f);
+    private float2 totalSum_b = (float2)(0.0f);
+
+    for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) {
+        uint sb = k / 8;
+        uint j  = k % 8;
+
+        // pair a scales/mins
+        half2 d_a  = src0_d[gid_a + sb * LINE_STRIDE_A];
+        half2 dm_a = src0_m[gid_a + sb * LINE_STRIDE_A];
+        global const uchar * sc0a = src0_s + sb * 12 * M + 2 * gid_a;
+        global const uchar * sc1a = sc0a + 1;
+        uchar sv0a, mn0a, sv1a, mn1a;
+        get_scale_min_k4(j, sc0a, M, &sv0a, &mn0a, mask_d6, mask_d4, mask_hi2);
+        get_scale_min_k4(j, sc1a, M, &sv1a, &mn1a, mask_d6, mask_d4, mask_hi2);
+        regS_a = convert_half2(convert_float2(d_a)  * convert_float2((uchar2)(sv0a, sv1a)));
+        regM_a = convert_half2(convert_float2(dm_a) * convert_float2((uchar2)(mn0a, mn1a)));
+
+        // pair b scales/mins
+        half2 d_b  = src0_d[gid_b + sb * LINE_STRIDE_A];
+        half2 dm_b = src0_m[gid_b + sb * LINE_STRIDE_A];
+        global const uchar * sc0b = src0_s + sb * 12 * M + 2 * gid_b;
+        global const uchar * sc1b = sc0b + 1;
+        uchar sv0b, mn0b, sv1b, mn1b;
+        get_scale_min_k4(j, sc0b, M, &sv0b, &mn0b, mask_d6, mask_d4, mask_hi2);
+        get_scale_min_k4(j, sc1b, M, &sv1b, &mn1b, mask_d6, mask_d4, mask_hi2);
+        regS_b = convert_half2(convert_float2(d_b)  * convert_float2((uchar2)(sv0b, sv1b)));
+        regM_b = convert_half2(convert_float2(dm_b) * convert_float2((uchar2)(mn0b, mn1b)));
+
+        // activation: load once, reuse for both pairs
+        if (slid < 4) {
+            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
+            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
+        }
+
+        // pair a (own block so _lo sees the shared_y declared by _hi)
+        {
+            regA.s0 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+            regA.s1 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+            regA.s2 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+            regA.s3 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+            dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB);
+#else
+            dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB);
+#endif
+            regA.s0 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+            regA.s1 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+            regA.s2 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+            regA.s3 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+            dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB);
+#else
+            dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB);
+#endif
+        }
+
+        // pair b
+        {
+            regA.s0 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+            regA.s1 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+            regA.s2 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+            regA.s3 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+            dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB);
+#else
+            dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB);
+#endif
+            regA.s0 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+            regA.s1 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+            regA.s2 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+            regA.s3 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+            dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB);
+#else
+            dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB);
+#endif
+        }
+    }
+
+    // reduce 4 outputs (a.s0, a.s1, b.s0, b.s1) across the 4 subgroups
+    local float4 reduceLM[SUBGROUP_SIZE * 3];
+    float4 acc = (float4)(totalSum_a.s0, totalSum_a.s1, totalSum_b.s0, totalSum_b.s1);
+    if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; }
+    if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; }
+    if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; }
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (groupId == 0) {
+        acc += reduceLM[SUBGROUP_SIZE * 0 + slid];
+        acc += reduceLM[SUBGROUP_SIZE * 1 + slid];
+        acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
+        dst = (global float*)((global char*)dst + offsetd);
+        // The dispatch rounds ne01/4 up to the subgroup width, so the tail
+        // quads past the last row must not store (they wrote 128 rows past
+        // dst on every ne01 % 256 == 128 vocab, e.g. 151936).
+        if (gid * 4 + 3 < (uint)ne01) {
+            vstore4(acc, 0, &(dst[gid * 4]));
+        }
+    }
+}
@@ -0,0 +1,118 @@
+// Tiled-wide q4_K GEMV for the long-vocab lm_head/embed (decode path).
+//
+// Pairs with kernel_convert_block_q4_k_tiled_ns (cvt.cl): the weights are laid
+// out CANONICALLY (4-bit code in element order e in [0,256)) and TILED by 64
+// output rows so the 64-thread lane group coalesces every weight load. Both the
+// pack (convert) and the unpack (here) are owned by us -> correct by
+// construction vs the reference ggml q4_K dequant. Same structure as the q6_K
+// tiled GEMV; the only differences are the 4-bit dequant and the q4_K
+// scale/min decode (get_scale_min_k4 from the packed 12-byte block).
+//
+// One work-item produces one output row. WG = {64 lanes, 4 subgroups}: the 64
+// lanes cover the 64 rows of one tile (coalesced uint4 reads), the 4 subgroups
+// split the K-blocks and reduce through __local at the end. Weights read from
+// __global (lm_head is streamed once per token; texture cache caps it below the
+// coalesced-global rate).
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define QK_K       256
+#define NSUBGROUPS 4
+#define TILE_ROWS  64
+
+// Decode one q4_K sub-block scale + min from the packed 12-byte block.
+// Identical to the o4 kernel's helper (masks hard-coded: d6=0x3F, d4=0x0F, hi2=0xC0).
+inline void q4k_scale_min(int j, __global const uchar * q, uchar * d, uchar * m) {
+    if (j < 4) {
+        *d = q[j]   & 0x3F;
+        *m = q[j+4] & 0x3F;
+    } else {
+        *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2);
+        *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2);
+    }
+}
+
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_tiled(
+    __global uint4 * src0_q,    // tiled: 8 uint4 granules / superblock (4-bit codes)
+    __global half  * src0_d,    // tiled: 1 half / superblock
+    __global half  * src0_dm,   // tiled: 1 half / superblock
+    __global uchar * src0_s,    // tiled: 12 bytes / superblock (packed scales)
+    read_only image1d_buffer_t src1,   // activation (RGBA f32)
+    global float * dst,
+    ulong offsetd,
+    int ne00,
+    int ne01
+) {
+    int grp = get_local_id(1);          // subgroup index 0..3 (splits K)
+    int row = get_global_id(0);         // output row along ne01
+    int rt  = row / TILE_ROWS;
+    int rit = row % TILE_ROWS;
+
+    int nb = ne00 / QK_K;               // superblocks per row
+
+    float acc = 0.0f;
+
+    for (int sb = grp; sb < nb; sb += NSUBGROUPS) {
+        int tile_blk = rt * nb + sb;    // ne02 == 1 for lm_head/embed
+
+        float dval  = (float)src0_d [tile_blk * TILE_ROWS + rit];
+        float dmval = (float)src0_dm[tile_blk * TILE_ROWS + rit];
+
+        // decode the 8 sub-block (scale, min) pairs
+        __global uchar * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 12;
+        float scale[8], minv[8];
+        #pragma unroll
+        for (int is = 0; is < 8; ++is) {
+            uchar sd, sm;
+            q4k_scale_min(is, sc, &sd, &sm);
+            scale[is] = dval  * (float)sd;
+            minv[is]  = dmval * (float)sm;
+        }
+
+        // 32 uints of 4-bit codes (8 codes/uint), e-order
+        uint q[32];
+        #pragma unroll
+        for (int g = 0; g < 8; ++g) {
+            uint4 v = src0_q[(tile_blk * 8 + g) * TILE_ROWS + rit];
+            q[g*4+0] = v.x; q[g*4+1] = v.y; q[g*4+2] = v.z; q[g*4+3] = v.w;
+        }
+
+        // dequant 256 codes in canonical e-order, MAC with activation.
+        int act_base = sb * 64;         // activation float4 pixel base (256/4)
+        #pragma unroll
+        for (int e4 = 0; e4 < 64; ++e4) {
+            float4 a = read_imagef(src1, act_base + e4);
+            #pragma unroll
+            for (int t = 0; t < 4; ++t) {
+                int  e    = e4 * 4 + t;
+                uint code = (q[e >> 3] >> ((e & 7) * 4)) & 0xF;
+                int  is   = e >> 5;     // sub-block index = e/32
+                float av  = (t == 0) ? a.x : (t == 1) ? a.y : (t == 2) ? a.z : a.w;
+                acc += ((float)code * scale[is] - minv[is]) * av;
+            }
+        }
+    }
+
+    // reduce across the NSUBGROUPS subgroups (same rit, different K-subset)
+    local float reduce_lm[NSUBGROUPS * TILE_ROWS];
+    reduce_lm[grp * TILE_ROWS + rit] = acc;
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (grp == 0) {
+        float total = reduce_lm[0 * TILE_ROWS + rit]
+                    + reduce_lm[1 * TILE_ROWS + rit]
+                    + reduce_lm[2 * TILE_ROWS + rit]
+                    + reduce_lm[3 * TILE_ROWS + rit];
+        dst = (global float*)((global char*)dst + offsetd);
+        dst[row] = total;
+    }
+}
@@ -329,3 +329,125 @@ kernel void kernel_gemv_noshuffle_q5_k_f32(
        if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
    }
 }
+
+// Multi-column (N in [2..4]) variant of the q5_K decode GEMV (spec/MTP verify) =
+// q4_K mc3 + the high-bit qh plane (regH). n_cols = 2..4 (drafted + bonus); routes
+// the small-batch verify OFF the gemm_noshuffle_q5_k dead-zone. n_cols==3 is byte-
+// identical to the original mc3 (col3 disabled, float8 slots 6/7 stay zero).
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+#define MC_DQ5_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi
+#define MC_DQ5_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo
+#else
+#define MC_DQ5_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi
+#define MC_DQ5_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo
+#endif
+#define MC_COL_Q5K(ts, c) \
+    { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE     + slid*2 + k*8); \
+                      regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \
+      MC_DQ5_HI(ts, as_ushort8(regA_hi), as_uchar8(regH), regS, regM, regB); \
+      MC_DQ5_LO(ts, as_ushort8(regA_lo), as_uchar8(regH), regS, regM, regB); }
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q5_k_f32_mc3(
+        read_only  image1d_buffer_t src0_q,
+        read_only  image1d_buffer_t src0_qh,
+        global half2  * src0_d,
+        global half2  * src0_m,
+        global uchar  * src0_s,
+        read_only  image1d_buffer_t src1,
+        global float * dst,
+        ulong offsetd,
+        int ne00,
+        int ne01,
+        uchar mask_d6,
+        uchar mask_d4,
+        uchar mask_hi2,
+        int n_cols)
+{
+    uint groupId = get_local_id(1);
+    uint gid     = get_global_id(0);
+    ushort slid  = get_sub_group_local_id();
+
+    uint K = ne00;
+    uint M = ne01;
+
+    uint LINE_STRIDE_A     = M / 2;
+    uint BLOCK_STRIDE_A    = NSUBGROUPS * M;
+    uint LINE_STRIDE_A_QH  = M / 2;
+    uint BLOCK_STRIDE_A_QH = NSUBGROUPS * M / 2;
+    uint scales_per_row    = (K / QK_K) * 12;
+    uint COL_STRIDE        = K / 4;   // float4 pixels per activation column
+
+    private uint4   regA_hi, regA_lo;
+    private ushort4 regH;
+    private half2   regS, regM;
+    private float8  regB;
+
+    private float2 ts0 = (float2)(0.0f);
+    private float2 ts1 = (float2)(0.0f);
+    private float2 ts2 = (float2)(0.0f);
+    private float2 ts3 = (float2)(0.0f);
+
+    for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) {
+        uint sb = k / 8;
+        uint j  = k % 8;
+
+        half2 d   = src0_d[gid + sb * LINE_STRIDE_A];
+        half2 dm  = src0_m[gid + sb * LINE_STRIDE_A];
+
+        global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12;
+        global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12;
+
+        uchar sv0, mn0, sv1, mn1;
+        get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+        get_scale_min_k4(j, sc1, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+
+        regS = convert_half2(convert_float2(d)  * convert_float2((uchar2)(sv0, sv1)));
+        regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));
+
+        // high-bit plane + weights loaded ONCE, reused across the columns
+        regH.s0 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x);
+        regH.s1 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x);
+        regH.s2 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x);
+        regH.s3 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x);
+
+        regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+        MC_COL_Q5K(ts0, 0);
+        MC_COL_Q5K(ts1, 1);
+        if (n_cols > 2) MC_COL_Q5K(ts2, 2);
+        if (n_cols > 3) MC_COL_Q5K(ts3, 3);
+    }
+
+    // cross-subgroup reduce: pack the (up to 4) columns' float2 into a float8.
+    local float8 reduceLM[SUBGROUP_SIZE * 3];
+    float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1);
+    if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; }
+    if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; }
+    if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; }
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (groupId == 0) {
+        acc += reduceLM[SUBGROUP_SIZE * 0 + slid];
+        acc += reduceLM[SUBGROUP_SIZE * 1 + slid];
+        acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
+        dst = (global float*)((global char*)dst + offsetd);
+        // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
+        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+        if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+        if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+    }
+}
+#undef MC_COL_Q5K
+#undef MC_DQ5_HI
+#undef MC_DQ5_LO
@@ -296,3 +296,114 @@ kernel void kernel_gemv_noshuffle_q6_K_f32(
        if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = total_sum.s1;
    }
 }
+
+// Multi-column (N=3) q6_K decode GEMV for the spec/MTP verify batch. Same idea
+// as the q4_K mc3: stay on the efficient GEMV path (subgroup broadcast, no
+// transpose) instead of the transposed-GEMM dead-zone. Each K-block's weights
+// (ql/qh, hi+lo) are loaded ONCE and reused across all 3 activation columns.
+// Per-column accumulation is independent and identical to 3 standalone GEMVs
+// => byte-identical; does NOT perturb the lm_head logits / spec accept rate.
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q6_K_f32_mc3(
+    read_only image1d_buffer_t src0_ql,
+    read_only image1d_buffer_t src0_qh,
+    global half2 * src0_s,
+    global half2 * src0_d,
+    read_only image1d_buffer_t src1,
+    global float * dst,
+    ulong offsetd,
+    int ne00,
+    int ne01
+) {
+    int grp  = get_local_id(1);
+    int gid  = get_global_id(0);
+    ushort slid = get_sub_group_local_id();
+
+    int nb = ne00 / 32;
+    int line_stride_a  = ne01 / 2;
+    int block_stride_a = NSUBGROUPS * ne01;
+    int COL_STRIDE     = ne00 / 4;   // float4 pixels per activation column
+
+    uint4   ql_hi, ql_lo;
+    ushort4 qh_hi, qh_lo;
+    half2   reg_d;
+    char4   reg_s;
+    float8  reg_b;
+
+    float2  ts0 = 0.0f, ts1 = 0.0f, ts2 = 0.0f;
+
+    for (int k = grp; k < nb; k += NSUBGROUPS) {
+        reg_d = src0_d[gid + k/8 * line_stride_a];
+        reg_s = as_char4(src0_s[gid + k * line_stride_a]);
+
+        // weights loaded ONCE (hi: blocks 0-3, lo: blocks 4-7), reused x3 cols
+        ql_hi.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*0).x;
+        ql_hi.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*1).x;
+        ql_hi.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*2).x;
+        ql_hi.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*3).x;
+        qh_hi.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*0).x);
+        qh_hi.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*1).x);
+        qh_hi.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*2).x);
+        qh_hi.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*3).x);
+
+        ql_lo.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*4).x;
+        ql_lo.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*5).x;
+        ql_lo.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*6).x;
+        ql_lo.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*7).x;
+        qh_lo.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*4).x);
+        qh_lo.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*5).x);
+        qh_lo.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*6).x);
+        qh_lo.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*7).x);
+
+        // Per-column: load only this column's activation (single reg_b live) ->
+        // 1/3 the activation register pressure, cutting the private-mem spill.
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+        { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0*COL_STRIDE + 0 + slid*2 + k*8);
+                          reg_b.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantize_block_acc_bcast_8_hi(ts0, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+          dequantize_block_acc_bcast_8_lo(ts0, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+        { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 1*COL_STRIDE + 0 + slid*2 + k*8);
+                          reg_b.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantize_block_acc_bcast_8_hi(ts1, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+          dequantize_block_acc_bcast_8_lo(ts1, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+        { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 2*COL_STRIDE + 0 + slid*2 + k*8);
+                          reg_b.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantize_block_acc_bcast_8_hi(ts2, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+          dequantize_block_acc_bcast_8_lo(ts2, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+#else
+        { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0*COL_STRIDE + 0 + slid*2 + k*8);
+                          reg_b.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantize_block_acc_bcast_1_hi(ts0, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+          dequantize_block_acc_bcast_1_lo(ts0, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+        { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 1*COL_STRIDE + 0 + slid*2 + k*8);
+                          reg_b.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantize_block_acc_bcast_1_hi(ts1, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+          dequantize_block_acc_bcast_1_lo(ts1, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+        { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 2*COL_STRIDE + 0 + slid*2 + k*8);
+                          reg_b.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+          dequantize_block_acc_bcast_1_hi(ts2, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+          dequantize_block_acc_bcast_1_lo(ts2, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+#endif
+    }
+
+    local float8 reduce_lm[SUBGROUP_SIZE * 3];
+    float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, 0.0f, 0.0f);
+    if (grp == 1) { reduce_lm[SUBGROUP_SIZE*0 + slid] = acc; }
+    if (grp == 2) { reduce_lm[SUBGROUP_SIZE*1 + slid] = acc; }
+    if (grp == 3) { reduce_lm[SUBGROUP_SIZE*2 + slid] = acc; }
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (grp == 0) {
+        acc += reduce_lm[SUBGROUP_SIZE*0 + slid];
+        acc += reduce_lm[SUBGROUP_SIZE*1 + slid];
+        acc += reduce_lm[SUBGROUP_SIZE*2 + slid];
+        dst = (global float*)((global char*)dst + offsetd);
+        // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row
+        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0*ne01 + gid*2]));
+        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1*ne01 + gid*2]));
+        vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2*ne01 + gid*2]));
+    }
+}
@@ -0,0 +1,372 @@
+// 4-output-per-WI variant of kernel_gemv_noshuffle_q6_K_f32.
+// Each WI now produces 4 consecutive outputs (output quad). The activation
+// fetch (reg_b) is shared across all 4 outputs, doubling per-WI ALU per
+// activation broadcast and halving the WG count vs the 2-output kernel.
+//
+// Implementation: each K-block we fetch TWO sets of (scales + ql + qh)
+// — one for the low pair (rows 0,1 of the quad) and one for the high pair
+// (rows 2,3) — and invoke the existing 2-output dequant macros twice
+// against the *same* reg_b. Identical data layout to the 2-output kernel,
+// so the host only needs to halve the grid and double the gid-to-output
+// mapping.
+//
+// Opt-in via the host dispatch when GGML_OPENCL_Q6K_GEMV_O4=1.
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define NSUBGROUPS 4
+#define SUBGROUP_SIZE 64
+
+// Macros are identical to the 2-output kernel — they accept `total_sum` as
+// a parameter so we can call them twice (once per pair) against different
+// accumulators against the same reg_b.
+#define dequantize_block_acc_bcast_8_hi(total_sum, bits4, bits2, cs, y) \
+    float8 shared_y; \
+    shared_y = sub_group_broadcast(y, 0); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x000F)      ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s0; \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >>  4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s1; \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >>  8) | ((bits2.s0 & 0x30)     )) - 32.f) * cs.s0 * shared_y.s2; \
+    total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s3; \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x000F)      ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s4; \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >>  4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s5; \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >>  8) | ((bits2.s2 & 0x30)     )) - 32.f) * cs.s0 * shared_y.s6; \
+    total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s7; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x000F)      ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s0; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >>  4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s1; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >>  8) | ((bits2.s1 & 0x30)     )) - 32.f) * cs.s2 * shared_y.s2; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s3; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x000F)      ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s4; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >>  4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s5; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >>  8) | ((bits2.s3 & 0x30)     )) - 32.f) * cs.s2 * shared_y.s6; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s7; \
+    shared_y = sub_group_broadcast(y, 1); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x000F)      ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s0; \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >>  4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s1; \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >>  8) | ((bits2.s4 & 0x30)     )) - 32.f) * cs.s0 * shared_y.s2; \
+    total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s3; \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x000F)      ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s4; \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >>  4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s5; \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >>  8) | ((bits2.s6 & 0x30)     )) - 32.f) * cs.s0 * shared_y.s6; \
+    total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s7; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x000F)      ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s0; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >>  4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s1; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >>  8) | ((bits2.s5 & 0x30)     )) - 32.f) * cs.s2 * shared_y.s2; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s3; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x000F)      ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s4; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >>  4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s5; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >>  8) | ((bits2.s7 & 0x30)     )) - 32.f) * cs.s2 * shared_y.s6; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s7; \
+
+#define dequantize_block_acc_bcast_8_lo(total_sum, bits4, bits2, cs, y) \
+    shared_y = sub_group_broadcast(y, 2); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x000F)      ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s0; \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >>  4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s1; \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >>  8) | ((bits2.s0 & 0x30)     )) - 32.f) * cs.s1 * shared_y.s2; \
+    total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s3; \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x000F)      ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s4; \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >>  4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s5; \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >>  8) | ((bits2.s2 & 0x30)     )) - 32.f) * cs.s1 * shared_y.s6; \
+    total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s7; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x000F)      ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s0; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >>  4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s1; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >>  8) | ((bits2.s1 & 0x30)     )) - 32.f) * cs.s3 * shared_y.s2; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s3; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x000F)      ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s4; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >>  4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s5; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >>  8) | ((bits2.s3 & 0x30)     )) - 32.f) * cs.s3 * shared_y.s6; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s7; \
+    shared_y = sub_group_broadcast(y, 3); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x000F)      ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s0; \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >>  4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s1; \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >>  8) | ((bits2.s4 & 0x30)     )) - 32.f) * cs.s1 * shared_y.s2; \
+    total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s3; \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x000F)      ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s4; \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >>  4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s5; \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >>  8) | ((bits2.s6 & 0x30)     )) - 32.f) * cs.s1 * shared_y.s6; \
+    total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s7; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x000F)      ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s0; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >>  4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s1; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >>  8) | ((bits2.s5 & 0x30)     )) - 32.f) * cs.s3 * shared_y.s2; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s3; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x000F)      ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s4; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >>  4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s5; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >>  8) | ((bits2.s7 & 0x30)     )) - 32.f) * cs.s3 * shared_y.s6; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s7; \
+
+#define dequantize_block_acc_bcast_1_hi(total_sum, bits4, bits2, cs, y) \
+    float shared_y; \
+    shared_y = sub_group_broadcast(y.s0, 0); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x000F)      ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x000F)      ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s1, 0); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >>  4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >>  4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s2, 0); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >>  8) | ((bits2.s0 & 0x30)     )) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >>  8) | ((bits2.s1 & 0x30)     )) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s3, 0); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s4, 0); \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x000F)      ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x000F)      ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s5, 0); \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >>  4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >>  4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s6, 0); \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >>  8) | ((bits2.s2 & 0x30)     )) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >>  8) | ((bits2.s3 & 0x30)     )) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s7, 0); \
+    total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s0, 1); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x000F)      ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x000F)      ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s1, 1); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >>  4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >>  4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s2, 1); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >>  8) | ((bits2.s4 & 0x30)     )) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >>  8) | ((bits2.s5 & 0x30)     )) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s3, 1); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s4, 1); \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x000F)      ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x000F)      ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s5, 1); \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >>  4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >>  4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s6, 1); \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >>  8) | ((bits2.s6 & 0x30)     )) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >>  8) | ((bits2.s7 & 0x30)     )) - 32.f) * cs.s2 * shared_y; \
+    shared_y = sub_group_broadcast(y.s7, 1); \
+    total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \
+
+#define dequantize_block_acc_bcast_1_lo(total_sum, bits4, bits2, cs, y) \
+    shared_y = sub_group_broadcast(y.s0, 2); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x000F)      ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x000F)      ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s1, 2); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >>  4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >>  4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s2, 2); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >>  8) | ((bits2.s0 & 0x30)     )) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >>  8) | ((bits2.s1 & 0x30)     )) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s3, 2); \
+    total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s4, 2); \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x000F)      ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x000F)      ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s5, 2); \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >>  4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >>  4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s6, 2); \
+    total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >>  8) | ((bits2.s2 & 0x30)     )) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >>  8) | ((bits2.s3 & 0x30)     )) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s7, 2); \
+    total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s0, 3); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x000F)      ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x000F)      ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s1, 3); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >>  4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >>  4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s2, 3); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >>  8) | ((bits2.s4 & 0x30)     )) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >>  8) | ((bits2.s5 & 0x30)     )) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s3, 3); \
+    total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s4, 3); \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x000F)      ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x000F)      ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s5, 3); \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >>  4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >>  4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s6, 3); \
+    total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >>  8) | ((bits2.s6 & 0x30)     )) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >>  8) | ((bits2.s7 & 0x30)     )) - 32.f) * cs.s3 * shared_y; \
+    shared_y = sub_group_broadcast(y.s7, 3); \
+    total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \
+    total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \
+
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+// Q6K_O4_GLOBAL: read the (read-once-per-token, no-reuse) lm_head/embed weights
+// from __global coalesced instead of image1d_buffer. The texture cache caps the
+// streaming (no-reuse) lm_head read bandwidth; global coalesced reaches the
+// higher rate the rest of the model gets. src1 (activation) stays an image (it IS reused via
+// the cross-subgroup broadcast).
+#ifdef Q6K_O4_GLOBAL
+#define Q6K_O4_NAME kernel_gemv_noshuffle_q6_K_f32_o4_global
+#define QL_ARG __global uint * src0_ql
+#define QH_ARG __global half * src0_qh
+#define RD_QL(b,i) (b[i])
+#define RD_QH(b,i) as_ushort(b[i])
+#else
+#define Q6K_O4_NAME kernel_gemv_noshuffle_q6_K_f32_o4
+#define QL_ARG read_only image1d_buffer_t src0_ql
+#define QH_ARG read_only image1d_buffer_t src0_qh
+#define RD_QL(b,i) (read_imageui(b,i).x)
+#define RD_QH(b,i) as_ushort(read_imageh(b,i).x)
+#endif
+kernel void Q6K_O4_NAME(
+    QL_ARG,
+    QH_ARG,
+    global half2 * src0_s,
+    global half2 * src0_d,
+    read_only image1d_buffer_t src1,
+    global float * dst,
+    ulong offsetd,
+    int ne00,
+    int ne01
+) {
+    int grp = get_local_id(1);
+    int gid = get_global_id(0);              // 4-output-quad index
+    ushort slid = get_sub_group_local_id();
+
+    // Map quad index to the two pair-indices the existing 2-output access
+    // pattern uses (consecutive output pairs along ne01). NB: the two pairs are
+    // kept ADJACENT (gid*2, gid*2+1) on purpose -- a "stride-1" split (pairs
+    // ne01/4 apart) is slower because two distant cache-line streams have worse
+    // locality than the adjacent pair whose reads interleave into the same lines
+    // each iteration.
+    int gid_a = gid * 2;
+    int gid_b = gid * 2 + 1;
+
+    int nb = ne00 / 32;
+
+    uint4   reg_a_l_a, reg_a_l_b;
+    ushort4 reg_a_h_a, reg_a_h_b;
+    half2   reg_d_a,   reg_d_b;
+    char4   reg_s_a,   reg_s_b;
+    float8  reg_b;
+
+    float2 total_sum_a = 0.0f;
+    float2 total_sum_b = 0.0f;
+
+    int line_stride_a = ne01 / 2;
+    int block_stride_a = NSUBGROUPS * ne01;
+
+    for (int k = grp; k < nb; k += NSUBGROUPS) {
+        reg_d_a = src0_d[gid_a + k/8 * line_stride_a];
+        reg_d_b = src0_d[gid_b + k/8 * line_stride_a];
+        reg_s_a = as_char4(src0_s[gid_a + k * line_stride_a]);
+        reg_s_b = as_char4(src0_s[gid_b + k * line_stride_a]);
+        // Precompute the loop-invariant combined scale (sub-block scale * super-block d)
+        // once per pair instead of re-multiplying it for every one of the 256 elements.
+        float4 cs_a = (float4)((float)reg_s_a.s0*(float)reg_d_a.s0, (float)reg_s_a.s1*(float)reg_d_a.s0,
+                               (float)reg_s_a.s2*(float)reg_d_a.s1, (float)reg_s_a.s3*(float)reg_d_a.s1);
+        float4 cs_b = (float4)((float)reg_s_b.s0*(float)reg_d_b.s0, (float)reg_s_b.s1*(float)reg_d_b.s0,
+                               (float)reg_s_b.s2*(float)reg_d_b.s1, (float)reg_s_b.s3*(float)reg_d_b.s1);
+
+        if (slid < 4) {
+            reg_b.s0123 = read_imagef(src1, 0 + slid*2 + k*8);
+            reg_b.s4567 = read_imagef(src1, 1 + slid*2 + k*8);
+        }
+
+        // Pair a (output rows gid_a*2, gid_a*2+1): read hi+lo then dequant
+        // both in one block so the `_lo` macro can see the `shared_y` that
+        // `_hi` declared. Pair b follows in its own block — fresh shared_y.
+        {
+            reg_a_l_a.s0 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*0);
+            reg_a_l_a.s1 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*1);
+            reg_a_l_a.s2 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*2);
+            reg_a_l_a.s3 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*3);
+            reg_a_h_a.s0 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*0);
+            reg_a_h_a.s1 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*1);
+            reg_a_h_a.s2 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*2);
+            reg_a_h_a.s3 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*3);
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+            dequantize_block_acc_bcast_8_hi(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b);
+#else
+            dequantize_block_acc_bcast_1_hi(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b);
+#endif
+
+            reg_a_l_a.s0 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*4);
+            reg_a_l_a.s1 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*5);
+            reg_a_l_a.s2 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*6);
+            reg_a_l_a.s3 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*7);
+            reg_a_h_a.s0 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*4);
+            reg_a_h_a.s1 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*5);
+            reg_a_h_a.s2 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*6);
+            reg_a_h_a.s3 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*7);
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+            dequantize_block_acc_bcast_8_lo(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b);
+#else
+            dequantize_block_acc_bcast_1_lo(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b);
+#endif
+        }
+
+        {
+            reg_a_l_b.s0 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*0);
+            reg_a_l_b.s1 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*1);
+            reg_a_l_b.s2 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*2);
+            reg_a_l_b.s3 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*3);
+            reg_a_h_b.s0 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*0);
+            reg_a_h_b.s1 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*1);
+            reg_a_h_b.s2 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*2);
+            reg_a_h_b.s3 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*3);
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+            dequantize_block_acc_bcast_8_hi(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b);
+#else
+            dequantize_block_acc_bcast_1_hi(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b);
+#endif
+
+            reg_a_l_b.s0 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*4);
+            reg_a_l_b.s1 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*5);
+            reg_a_l_b.s2 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*6);
+            reg_a_l_b.s3 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*7);
+            reg_a_h_b.s0 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*4);
+            reg_a_h_b.s1 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*5);
+            reg_a_h_b.s2 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*6);
+            reg_a_h_b.s3 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*7);
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+            dequantize_block_acc_bcast_8_lo(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b);
+#else
+            dequantize_block_acc_bcast_1_lo(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b);
+#endif
+        }
+    }
+
+    // Cross-subgroup reduce. Same shape as the 2-output kernel but with the
+    // pair-a and pair-b accumulators concatenated into a single float4.
+    local float4 reduce_lm[SUBGROUP_SIZE * 3];
+    float4 acc = (float4)(total_sum_a.s0, total_sum_a.s1, total_sum_b.s0, total_sum_b.s1);
+    if (grp == 1) { reduce_lm[SUBGROUP_SIZE*0 + slid] = acc; }
+    if (grp == 2) { reduce_lm[SUBGROUP_SIZE*1 + slid] = acc; }
+    if (grp == 3) { reduce_lm[SUBGROUP_SIZE*2 + slid] = acc; }
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (grp == 0) {
+        acc += reduce_lm[SUBGROUP_SIZE*0 + slid];
+        acc += reduce_lm[SUBGROUP_SIZE*1 + slid];
+        acc += reduce_lm[SUBGROUP_SIZE*2 + slid];
+        dst = (global float*)((global char*)dst + offsetd);
+        // The dispatch rounds ne01/4 up to the subgroup width, so the tail
+        // quads past the last row must not store (they wrote 128 rows past
+        // dst on every ne01 % 256 == 128 vocab, e.g. 151936).
+        if (gid * 4 + 3 < (uint)ne01) {
+            vstore4(acc, 0, &(dst[gid * 4]));
+        }
+    }
+}
@@ -0,0 +1,196 @@
+// Tiled-wide q6_K GEMV for the long-vocab lm_head/embed (decode path).
+//
+// Pairs with kernel_convert_block_q6_k_tiled_ns (cvt.cl): the weights are laid
+// out CANONICALLY (6-bit code in element order e in [0,256)) and TILED by 64
+// output rows so the 64-thread lane group coalesces every weight load. Both the
+// pack (convert) and the unpack (here) are owned by us — correct by construction
+// against the reference ggml q6_K dequant, no bit-interleave reverse-engineering.
+//
+// One work-item produces one output row. A work-group is {64 lanes, 4 subgroups}:
+// the 64 lanes cover the 64 rows of one tile (coalesced reads), the 4 subgroups
+// split the K-blocks and reduce through __local at the end.
+//
+// Weights are read from __global (coalesced) rather than image1d_buffer: the
+// lm_head is read once per token with no reuse, and the Adreno texture cache
+// caps such a streaming read well below the coalesced-global rate
+// (see opencl_q6k_gemv_o4_shipped / x2-90 roofline notes).
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define NSUBGROUPS 4
+#define TILE_ROWS  64
+
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q6_K_f32_tiled(
+    __global uint4 * src0_ql,   // tiled: 8 uint4 granules / superblock
+    __global uint4 * src0_qh,   // tiled: 4 uint4 granules / superblock
+    __global char  * src0_s,    // tiled: 16 chars / superblock
+    __global half  * src0_d,    // tiled: 1 half  / superblock
+    read_only image1d_buffer_t src1,   // activation (RGBA f32)
+    global float * dst,
+    ulong offsetd,
+    int ne00,
+    int ne01
+) {
+    int grp = get_local_id(1);          // subgroup index 0..3 (splits K)
+    int row = get_global_id(0);         // output row along ne01
+    int rt  = row / TILE_ROWS;
+    int rit = row % TILE_ROWS;
+
+    int nb = ne00 / 256;                // superblocks per row
+
+    float acc = 0.0f;
+
+    for (int sb = grp; sb < nb; sb += NSUBGROUPS) {
+        int tile_blk = rt * nb + sb;    // ne02 == 1 for lm_head/embed
+
+        // d + 16 scales for this (row, superblock)
+        float dval = (float)src0_d[tile_blk * TILE_ROWS + rit];
+        __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16;
+
+        // 32 ql-uints (8 codes/uint) + 16 qh-uints (16 codes/uint)
+        uint ql[32];
+        uint qh[16];
+        #pragma unroll
+        for (int g = 0; g < 8; ++g) {
+            uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit];
+            ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w;
+        }
+        #pragma unroll
+        for (int g = 0; g < 4; ++g) {
+            uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit];
+            qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w;
+        }
+
+        // dequant 256 codes in canonical e-order, MAC with activation.
+        int act_base = sb * 64;         // activation float4 pixel base (256/4)
+        #pragma unroll
+        for (int e4 = 0; e4 < 64; ++e4) {
+            float4 a = read_imagef(src1, act_base + e4);
+            #pragma unroll
+            for (int t = 0; t < 4; ++t) {
+                int  e    = e4 * 4 + t;
+                uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF;
+                uint hi2  = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3;
+                int  code = (int)(low4 | (hi2 << 4)) - 32;
+                int  sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1);
+                float scale = (float)sc[sidx] * dval;
+                float av = (t == 0) ? a.x : (t == 1) ? a.y : (t == 2) ? a.z : a.w;
+                acc += (float)code * scale * av;
+            }
+        }
+    }
+
+    // reduce across the NSUBGROUPS subgroups (same rit, different K-subset)
+    local float reduce_lm[NSUBGROUPS * TILE_ROWS];
+    reduce_lm[grp * TILE_ROWS + rit] = acc;
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (grp == 0) {
+        float total = reduce_lm[0 * TILE_ROWS + rit]
+                    + reduce_lm[1 * TILE_ROWS + rit]
+                    + reduce_lm[2 * TILE_ROWS + rit]
+                    + reduce_lm[3 * TILE_ROWS + rit];
+        dst = (global float*)((global char*)dst + offsetd);
+        dst[row] = total;
+    }
+}
+
+// Multi-column (N=3) variant of the tiled q6_K decode GEMV, for the speculative/
+// MTP VERIFY lm_head/embed (ne1=3 = 2 drafts + 1 bonus). Identical tiled weight
+// layout + unpack as the ne1=1 kernel above; each WI computes 3 output columns,
+// streaming the (large) lm_head weight ONCE per superblock and reusing it across
+// the 3 verify activation columns (dequant once per code, MAC into 3 accs). This
+// is the lm_head analogue of the per-layer mc3 GEMV; the multiply order matches
+// the ne1=1 kernel, so each column is byte-identical to a standalone tiled GEMV.
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3(
+    __global uint4 * src0_ql,
+    __global uint4 * src0_qh,
+    __global char  * src0_s,
+    __global half  * src0_d,
+    read_only image1d_buffer_t src1,
+    global float * dst,
+    ulong offsetd,
+    int ne00,
+    int ne01
+) {
+    int grp = get_local_id(1);
+    int row = get_global_id(0);
+    int rt  = row / TILE_ROWS;
+    int rit = row % TILE_ROWS;
+
+    int nb = ne00 / 256;
+    int col_stride = ne00 / 4;          // activation float4 pixels per column
+
+    float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f;
+
+    for (int sb = grp; sb < nb; sb += NSUBGROUPS) {
+        int tile_blk = rt * nb + sb;
+
+        float dval = (float)src0_d[tile_blk * TILE_ROWS + rit];
+        __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16;
+
+        uint ql[32];
+        uint qh[16];
+        #pragma unroll
+        for (int g = 0; g < 8; ++g) {
+            uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit];
+            ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w;
+        }
+        #pragma unroll
+        for (int g = 0; g < 4; ++g) {
+            uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit];
+            qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w;
+        }
+
+        int act_base = sb * 64;
+        #pragma unroll
+        for (int e4 = 0; e4 < 64; ++e4) {
+            float4 a0 = read_imagef(src1, 0*col_stride + act_base + e4);
+            float4 a1 = read_imagef(src1, 1*col_stride + act_base + e4);
+            float4 a2 = read_imagef(src1, 2*col_stride + act_base + e4);
+            #pragma unroll
+            for (int t = 0; t < 4; ++t) {
+                int  e    = e4 * 4 + t;
+                uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF;
+                uint hi2  = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3;
+                int  code = (int)(low4 | (hi2 << 4)) - 32;
+                int  sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1);
+                float w   = (float)code * ((float)sc[sidx] * dval);  // dequant+scale once
+                float av0 = (t == 0) ? a0.x : (t == 1) ? a0.y : (t == 2) ? a0.z : a0.w;
+                float av1 = (t == 0) ? a1.x : (t == 1) ? a1.y : (t == 2) ? a1.z : a1.w;
+                float av2 = (t == 0) ? a2.x : (t == 1) ? a2.y : (t == 2) ? a2.z : a2.w;
+                acc0 += w * av0;
+                acc1 += w * av1;
+                acc2 += w * av2;
+            }
+        }
+    }
+
+    local float4 reduce_lm[NSUBGROUPS * TILE_ROWS];
+    reduce_lm[grp * TILE_ROWS + rit] = (float4)(acc0, acc1, acc2, 0.0f);
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (grp == 0) {
+        float4 total = reduce_lm[0 * TILE_ROWS + rit]
+                     + reduce_lm[1 * TILE_ROWS + rit]
+                     + reduce_lm[2 * TILE_ROWS + rit]
+                     + reduce_lm[3 * TILE_ROWS + rit];
+        dst = (global float*)((global char*)dst + offsetd);
+        // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row
+        dst[0*ne01 + row] = total.x;
+        dst[1*ne01 + row] = total.y;
+        dst[2*ne01 + row] = total.z;
+    }
+}
@@ -118,6 +118,87 @@
    elem = (char)((bits8.s7 & 0xFF000000) >> 24); \
    total_sums += convert_int(elem) * scale * shared_y; \

+// ============================================================================
+// Split-K variant for small-M decode GEMVs.
+// ----------------------------------------------------------------------------
+// The base kernel below puts one output row per lane and splits K only across
+// the N_SIMDGROUP subgroups of a single workgroup, so M=512 yields M/64 = 8
+// workgroups -- half the compute units on a 16-CU X2 sit idle, and the kernel
+// measures ~48 GB/s against the ~122 GB/s the larger projections reach in the
+// same graph. Here each (kslice, subgroup) pair reduces a disjoint set of
+// K-blocks into partial[kslice * M + row]; kernel_gemv_splitk_reduce_f32 (in
+// gemv_noshuffle_q4_k_f32.cl) sums the slices. Same operand order within a
+// slice as the base kernel; only the cross-slice grouping differs.
+//
+// Placed BEFORE the base kernel deliberately: on A6X no kernel may be defined
+// after one that uses a subgroup builtin, or it silently miscompiles.
+// ============================================================================
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+__kernel void kernel_gemv_noshuffle_q8_0_f32_splitk(
+        __read_only  image1d_buffer_t src0_q,   // quantized A (weights)
+        global half  * src0_d,                  // A scales
+        __read_only  image1d_buffer_t src1,     // B (activations)
+        global float * partial,                 // [ksplit * M], slice-major
+        int ne00,                               // K
+        int ne01)                               // M
+{
+    uint groupId = get_local_id(1);
+    uint gid     = get_global_id(0);
+    ushort slid  = get_sub_group_local_id();
+    uint nsg     = get_local_size(1);
+    uint ksplit  = get_num_groups(1);
+    uint kslice  = get_group_id(1);
+
+    uint K = ne00;
+    uint M = ne01;
+
+    uint LINE_STRIDE_A  = M;
+    uint BLOCK_STRIDE_A = 8 * M;   // physical, independent of the K-split
+
+    __private uint8  regA;
+    __private half   regS;
+    __private float8 regB;
+    __private float  totalSum = (float)(0.0f);
+
+    #pragma unroll 1
+    for (uint k = kslice * nsg + groupId; k < (K / QK8_0); k += ksplit * nsg) {
+        regS = src0_d[gid + k * LINE_STRIDE_A];
+        if (slid < 4) {
+            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
+            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
+        }
+        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+        dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB);
+    }
+
+    // Intra-workgroup reduce across this K-slice's subgroups. Sized for
+    // nsg <= 8; the host never dispatches more.
+    __local float reduceLM[SIMDGROUP_WIDTH * 7];
+    if (groupId > 0) {
+        reduceLM[SIMDGROUP_WIDTH * (groupId - 1) + slid] = totalSum;
+    }
+    barrier(CLK_LOCAL_MEM_FENCE);
+    if (groupId == 0) {
+        for (uint i = 0; i < nsg - 1; ++i) {
+            totalSum += reduceLM[SIMDGROUP_WIDTH * i + slid];
+        }
+        // x-grid is padded to CEIL_DIV(M,wave)*wave; guard the tail rows.
+        if (gid < M) {
+            partial[kslice * M + gid] = totalSum;
+        }
+    }
+}
+
 #ifdef ADRENO_GPU
 REQD_SUBGROUP_SIZE_64
 #endif
@@ -145,3 +145,52 @@ kernel void kernel_mul_mm_f32_f32_l4_lm(
        }
    }
 }
+
+// Multi-column f32 GEMV for the small-N (spec/MTP verify) batch. The tiled GEMM
+// above always computes a full BM x BN = 64 x 64 output tile, so at ne11=3 with a
+// skinny weight (e.g. GDN ssm_alpha/ssm_beta, M=32) it launches ONE under-occupied
+// workgroup at ~2.3% tile utilization. This kernel assigns one 64-thread workgroup
+// per output element (m,n): the 64 threads split the K reduction (float4) and
+// tree-reduce in __local (no subgroup ops -> portable). ne01*ne11 workgroups.
+// Weight row is re-read per column (N small -> negligible). Summation order differs
+// from the tiled GEMM (lane-strided + tree) -> f32-exact-ish, not bit-identical.
+kernel void kernel_gemv_f32_f32_mc(
+    global float * src0, ulong offset0,   // weight: row m at m*stride_a (elements)
+    global float * src1, ulong offset1,   // activations: col n at n*stride_b
+    global float * dst,  ulong offsetd,   // dst [M x N] col-major: (m,n) at n*stride_d+m
+    int ne00,        // K
+    int ne01,        // M
+    int ne11,        // N
+    int stride_a,    // weight row stride (elements) = K
+    int stride_b,    // activation col stride (elements) = K
+    int stride_d)    // dst column stride (elements) = M
+{
+    src0 = (global float*)((global char*)src0 + offset0);
+    src1 = (global float*)((global char*)src1 + offset1);
+    dst  = (global float*)((global char*)dst  + offsetd);
+
+    uint lane = get_local_id(0);            // 0..63
+    uint out  = get_global_id(1);           // 0 .. ne01*ne11 - 1
+    uint m = out % (uint)ne01;
+    uint n = out / (uint)ne01;
+
+    global float4 * wrow = (global float4*)(src0 + (ulong)m * (uint)stride_a);
+    global float4 * xcol = (global float4*)(src1 + (ulong)n * (uint)stride_b);
+    uint k4 = (uint)ne00 >> 2;
+
+    float acc = 0.0f;
+    for (uint k = lane; k < k4; k += 64) {
+        float4 w = wrow[k];
+        float4 x = xcol[k];
+        acc += w.s0*x.s0 + w.s1*x.s1 + w.s2*x.s2 + w.s3*x.s3;
+    }
+
+    local float red[64];
+    red[lane] = acc;
+    barrier(CLK_LOCAL_MEM_FENCE);
+    for (uint s = 32; s > 0; s >>= 1) {
+        if (lane < s) red[lane] += red[lane + s];
+        barrier(CLK_LOCAL_MEM_FENCE);
+    }
+    if (lane == 0) dst[(ulong)n * (uint)stride_d + m] = red[0];
+}
@@ -0,0 +1,306 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_subgroups
+#pragma OPENCL EXTENSION cl_intel_subgroups : enable
+#else
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#endif
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+// Multi-row f16xf32 GEMV for the DECODE path (single token, ne11*ne12 small).
+// The legacy kernel_mul_mat_f16_f32_1row runs ONE 64-lane subgroup per workgroup =
+// one output row per WG, which caps memory-level parallelism at roughly half of
+// LPDDR5x peak. This variant packs MROW subgroups per workgroup, each
+// computing a distinct output row, so a WG keeps 64*MROW loads in flight. The
+// activation column y (shared by every output row) is staged into __local ONCE per
+// WG and reused across the MROW rows, cutting redundant activation reads. Used for
+// the f16 attention projections (Q/K/V/O) and lm_head, which dominate decode.
+// Numerically equivalent to _1row (same f16->f32 widening, same float4 partial sums,
+// same subgroup-reduce order), so byte-identical to the per-op path.
+
+#define MROW 16
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow(
+        global char * src0,
+        ulong offset0,
+        global char * src1,
+        ulong offset1,
+        global float * dst,
+        ulong offsetd,
+        int ne00,
+        int ne01,
+        int ne02,
+        ulong nb00,
+        ulong nb01,
+        ulong nb02,
+        ulong nb03,
+        int ne10,
+        int ne11,
+        int ne12,
+        ulong nb10,
+        ulong nb11,
+        ulong nb12,
+        ulong nb13,
+        int ne0,
+        int ne1,
+        int r2,
+        int r3,
+        __local float * ysh
+) {
+    src0 = (global char*)((global char*)src0 + offset0);
+    src1 = (global char*)((global char*)src1 + offset1);
+    dst  = (global float*)((global char*)dst + offsetd);
+
+    int r0   = get_group_id(0) * MROW + get_local_id(1);  // output row
+    int r1   = get_group_id(1);                            // token (ne11)
+    int im   = get_group_id(2);
+    int lid  = get_sub_group_local_id();                   // 0..63
+    int nsg  = get_local_size(1);                          // == MROW
+
+    int i12 = im % ne12;
+    int i13 = im / ne12;
+
+    ulong offset_src1 = r1*nb11 + (i12)*nb12 + (i13)*nb13;
+    global float * y = (global float *) (src1 + offset_src1);
+
+    // Cooperatively stage the activation column (ne00 floats) into __local once per
+    // WG and reuse across the MROW rows. Staging is the actual win here: dropping it
+    // (each subgroup re-reading y from global) regresses below the 1-row kernel.
+    for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; i += nsg*get_sub_group_size()) {
+        ysh[i] = y[i];
+    }
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (r0 >= ne01) {
+        return;
+    }
+
+    ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
+    global half * x = (global half *) (src0 + offset_src0);
+
+    // The vector path below casts the row pointer to half4, which must be 8-byte aligned.
+    // A row address is r0*nb01 + ..., and a permuted or strided src0 leaves nb01/nb02/nb03
+    // unconstrained -- ne00 % 4 == 0 bounds the element count per row, not the byte stride
+    // between rows. Take the vector path only when this work-item's row is actually
+    // aligned; the scalar loop below has no such requirement.
+    const bool row_aligned = (((ulong) x) & 7) == 0;
+
+    float sumf = 0.0f;
+    if (ne00 < 128 || !row_aligned) {
+        for (int i = lid; i < ne00; i += get_sub_group_size()) {
+            sumf += (float) x[i] * ysh[i];
+        }
+        float all_sum = sub_group_reduce_add(sumf);
+        if (lid == 0) {
+            dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+        }
+    } else {
+        global half4 * x4 = (global half4 *) x;
+        __local float4 * ysh4 = (__local float4 *) ysh;
+        for (int i = lid; i < ne00/4; i += get_sub_group_size()) {
+            float4 yv = ysh4[i];
+            sumf += (float) x4[i].s0 * yv.s0;
+            sumf += (float) x4[i].s1 * yv.s1;
+            sumf += (float) x4[i].s2 * yv.s2;
+            sumf += (float) x4[i].s3 * yv.s3;
+        }
+        float all_sum = sub_group_reduce_add(sumf);
+        if (lid == 0) {
+            for (int i = 4*(ne00/4); i < ne00; ++i) {
+                all_sum += (float) x[i] * ysh[i];
+            }
+            dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+        }
+    }
+}
+
+// Register-blocked variant: each 64-lane subgroup accumulates RPT consecutive
+// output rows instead of one. The staged activation is reused across all RPT rows,
+// and each lane keeps RPT independent weight loads in flight per column step ->
+// more memory-level parallelism on the streaming f16 weight read (the BW limiter),
+// plus RPT fewer staging barriers per output row. Per-row reduction order is
+// identical to _mrow, so byte-identical to the per-op path. Dispatch guarantees
+// ne00 >= 128 and ne00 % 4 == 0, so only the half4 path is needed (no tail).
+#define MROW_RB_BODY(RPT)                                                                 \
+    src0 = (global char*)((global char*)src0 + offset0);                                  \
+    src1 = (global char*)((global char*)src1 + offset1);                                  \
+    dst  = (global float*)((global char*)dst + offsetd);                                  \
+    int r0b = (get_group_id(0) * get_local_size(1) + get_local_id(1)) * (RPT);            \
+    int r1  = get_group_id(1);                                                            \
+    int im  = get_group_id(2);                                                            \
+    int lid = get_sub_group_local_id();                                                   \
+    int nsg = get_local_size(1);                                                          \
+    int i12 = im % ne12;                                                                  \
+    int i13 = im / ne12;                                                                  \
+    ulong off_y = r1*nb11 + i12*nb12 + i13*nb13;                                          \
+    global float * y = (global float *) (src1 + off_y);                                   \
+    for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00;                    \
+         i += nsg*get_sub_group_size()) {                                                 \
+        ysh[i] = y[i];                                                                    \
+    }                                                                                     \
+    barrier(CLK_LOCAL_MEM_FENCE);                                                         \
+    __local float4 * ysh4 = (__local float4 *) ysh;                                       \
+    global half4 * xr[RPT];                                                               \
+    _Pragma("unroll")                                                                     \
+    for (int rr = 0; rr < (RPT); ++rr) {                                                  \
+        int row = r0b + rr;                                                               \
+        if (row > ne01 - 1) row = ne01 - 1;                                               \
+        xr[rr] = (global half4 *) (src0 + (ulong)row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); \
+    }                                                                                     \
+    float sumf[RPT];                                                                      \
+    _Pragma("unroll")                                                                     \
+    for (int rr = 0; rr < (RPT); ++rr) sumf[rr] = 0.0f;                                   \
+    for (int i = lid; i < ne00/4; i += get_sub_group_size()) {                            \
+        float4 yv = ysh4[i];                                                              \
+        _Pragma("unroll")                                                                 \
+        for (int rr = 0; rr < (RPT); ++rr) {                                              \
+            half4 xv = xr[rr][i];                                                         \
+            sumf[rr] += (float) xv.s0 * yv.s0 + (float) xv.s1 * yv.s1                      \
+                      + (float) xv.s2 * yv.s2 + (float) xv.s3 * yv.s3;                     \
+        }                                                                                 \
+    }                                                                                     \
+    _Pragma("unroll")                                                                     \
+    for (int rr = 0; rr < (RPT); ++rr) {                                                  \
+        float s = sub_group_reduce_add(sumf[rr]);                                         \
+        int row = r0b + rr;                                                               \
+        if (lid == 0 && row < ne01) {                                                     \
+            dst[im*ne1*ne0 + r1*ne0 + row] = s;                                           \
+        }                                                                                 \
+    }
+
+// half8 (128-bit) load variant: Adreno's load/store unit issues 128-bit
+// transactions, so half4 (64-bit) loads may leave the load path half-idle. This
+// processes 8 weight elements per lane per step via half8. Accumulation groups
+// elements in 8s rather than 4s, so it is NOT bit-identical to _1row (float add is
+// non-associative) -- experimental BW probe, gate on ne00 % 8 == 0.
+#define MROW_H8_BODY(RPT)                                                                 \
+    src0 = (global char*)((global char*)src0 + offset0);                                  \
+    src1 = (global char*)((global char*)src1 + offset1);                                  \
+    dst  = (global float*)((global char*)dst + offsetd);                                  \
+    int r0b = (get_group_id(0) * get_local_size(1) + get_local_id(1)) * (RPT);            \
+    int r1  = get_group_id(1);                                                            \
+    int im  = get_group_id(2);                                                            \
+    int lid = get_sub_group_local_id();                                                   \
+    int nsg = get_local_size(1);                                                          \
+    int i12 = im % ne12;                                                                  \
+    int i13 = im / ne12;                                                                  \
+    ulong off_y = r1*nb11 + i12*nb12 + i13*nb13;                                          \
+    global float * y = (global float *) (src1 + off_y);                                   \
+    for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00;                    \
+         i += nsg*get_sub_group_size()) {                                                 \
+        ysh[i] = y[i];                                                                    \
+    }                                                                                     \
+    barrier(CLK_LOCAL_MEM_FENCE);                                                         \
+    __local float4 * ysh4 = (__local float4 *) ysh;                                       \
+    global half8 * xr[RPT];                                                               \
+    _Pragma("unroll")                                                                     \
+    for (int rr = 0; rr < (RPT); ++rr) {                                                  \
+        int row = r0b + rr;                                                               \
+        if (row > ne01 - 1) row = ne01 - 1;                                               \
+        xr[rr] = (global half8 *) (src0 + (ulong)row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); \
+    }                                                                                     \
+    float sumf[RPT];                                                                      \
+    _Pragma("unroll")                                                                     \
+    for (int rr = 0; rr < (RPT); ++rr) sumf[rr] = 0.0f;                                   \
+    for (int i = lid; i < ne00/8; i += get_sub_group_size()) {                            \
+        float4 y0 = ysh4[2*i];                                                            \
+        float4 y1 = ysh4[2*i + 1];                                                        \
+        _Pragma("unroll")                                                                 \
+        for (int rr = 0; rr < (RPT); ++rr) {                                              \
+            half8 xv = xr[rr][i];                                                         \
+            sumf[rr] += (float) xv.s0 * y0.s0 + (float) xv.s1 * y0.s1                      \
+                      + (float) xv.s2 * y0.s2 + (float) xv.s3 * y0.s3                      \
+                      + (float) xv.s4 * y1.s0 + (float) xv.s5 * y1.s1                      \
+                      + (float) xv.s6 * y1.s2 + (float) xv.s7 * y1.s3;                     \
+        }                                                                                 \
+    }                                                                                     \
+    _Pragma("unroll")                                                                     \
+    for (int rr = 0; rr < (RPT); ++rr) {                                                  \
+        float s = sub_group_reduce_add(sumf[rr]);                                         \
+        int row = r0b + rr;                                                               \
+        if (lid == 0 && row < ne01) {                                                     \
+            dst[im*ne1*ne0 + r1*ne0 + row] = s;                                           \
+        }                                                                                 \
+    }
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow_h8(
+        global char * src0, ulong offset0,
+        global char * src1, ulong offset1,
+        global float * dst, ulong offsetd,
+        int ne00, int ne01, int ne02,
+        ulong nb00, ulong nb01, ulong nb02, ulong nb03,
+        int ne10, int ne11, int ne12,
+        ulong nb10, ulong nb11, ulong nb12, ulong nb13,
+        int ne0, int ne1, int r2, int r3,
+        __local float * ysh
+) {
+    MROW_H8_BODY(1)
+}
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow_h8r2(
+        global char * src0, ulong offset0,
+        global char * src1, ulong offset1,
+        global float * dst, ulong offsetd,
+        int ne00, int ne01, int ne02,
+        ulong nb00, ulong nb01, ulong nb02, ulong nb03,
+        int ne10, int ne11, int ne12,
+        ulong nb10, ulong nb11, ulong nb12, ulong nb13,
+        int ne0, int ne1, int r2, int r3,
+        __local float * ysh
+) {
+    MROW_H8_BODY(2)
+}
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow_r2(
+        global char * src0, ulong offset0,
+        global char * src1, ulong offset1,
+        global float * dst, ulong offsetd,
+        int ne00, int ne01, int ne02,
+        ulong nb00, ulong nb01, ulong nb02, ulong nb03,
+        int ne10, int ne11, int ne12,
+        ulong nb10, ulong nb11, ulong nb12, ulong nb13,
+        int ne0, int ne1, int r2, int r3,
+        __local float * ysh
+) {
+    MROW_RB_BODY(2)
+}
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow_r4(
+        global char * src0, ulong offset0,
+        global char * src1, ulong offset1,
+        global float * dst, ulong offsetd,
+        int ne00, int ne01, int ne02,
+        ulong nb00, ulong nb01, ulong nb02, ulong nb03,
+        int ne10, int ne11, int ne12,
+        ulong nb10, ulong nb11, ulong nb12, ulong nb13,
+        int ne0, int ne1, int r2, int r3,
+        __local float * ysh
+) {
+    MROW_RB_BODY(4)
+}
@@ -188,3 +188,182 @@ kernel void kernel_rms_norm_mul(
        y[i00] = (x[i00] * scale) * f[i00%(ne10/4)];
    }
 }
+
+//------------------------------------------------------------------------------
+// rms_norm + mul (norm weight) + add (residual), fused. Mirrors
+// kernel_rms_norm_mul with an extra residual operand src2: computes
+//   y = (rmsnorm(x) * w) + g
+// in one dispatch, removing one kernel launch + one global round-trip per
+// residual block (the dominant per-layer adjacency on Gemma matformers).
+//------------------------------------------------------------------------------
+kernel void kernel_rms_norm_mul_add(
+        global char * src0,
+        ulong offset0,
+        global char * src1,
+        ulong offset1,
+        global char * src2,
+        ulong offset2,
+        global char * dst,
+        ulong offsetd,
+        int ne00,
+        int ne01,
+        int ne02,
+        int ne03,
+        ulong nb01,
+        ulong nb02,
+        ulong nb03,
+        int ne10,
+        int ne11,
+        int ne12,
+        int ne13,
+        ulong nb11,
+        ulong nb12,
+        ulong nb13,
+        int ne20,
+        int ne21,
+        int ne22,
+        int ne23,
+        ulong nb21,
+        ulong nb22,
+        ulong nb23,
+        ulong nb1,
+        ulong nb2,
+        ulong nb3,
+        float eps,
+        local float * sum
+) {
+    src0 = src0 + offset0;
+    src1 = src1 + offset1;
+    src2 = src2 + offset2;
+    dst  = dst  + offsetd;
+
+    if (get_sub_group_id() == 0) {
+        sum[get_sub_group_local_id()] = 0.0f;
+    }
+
+    int i03 = get_group_id(2);
+    int i02 = get_group_id(1);
+    int i01 = get_group_id(0);
+
+    global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);
+    global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11);
+    global float4 * g = (global float4 *) (src2 + (i03%ne23)*nb23 + (i02%ne22)*nb22 + (i01%ne21)*nb21);
+
+    float sumf = 0;
+
+    for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
+        sumf += dot(x[i00], x[i00]);
+    }
+    sumf = sub_group_reduce_add(sumf);
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (get_sub_group_local_id() == 0) {
+        sum[get_sub_group_id()] = sumf;
+    }
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    sumf = sum[get_sub_group_local_id()];
+    sumf = sub_group_reduce_add(sumf);
+
+    float mean  = sumf / ne00;
+    float scale = 1.0f/sqrt(mean + eps);
+
+    global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1);
+    for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
+        y[i00] = (x[i00] * scale) * f[i00%(ne10/4)] + g[i00%(ne20/4)];
+    }
+}
+
+//------------------------------------------------------------------------------
+// rms_norm + mul(norm weight) + add(residual) + mul(scalar scale), fused.
+// Computes y = ((rmsnorm(x) * w) + g) * s, where s is a broadcast SCALAR (e.g.
+// Gemma-4 layer_output_scale). Folds the trailing per-layer l_out scale-mul into
+// the residual-norm kernel: one extra dispatch + global round-trip saved per
+// layer. src3 points at the single scale value.
+//------------------------------------------------------------------------------
+kernel void kernel_rms_norm_mul_add_scale(
+        global char * src0,
+        ulong offset0,
+        global char * src1,
+        ulong offset1,
+        global char * src2,
+        ulong offset2,
+        global char * src3,
+        ulong offset3,
+        global char * dst,
+        ulong offsetd,
+        int ne00,
+        int ne01,
+        int ne02,
+        int ne03,
+        ulong nb01,
+        ulong nb02,
+        ulong nb03,
+        int ne10,
+        int ne11,
+        int ne12,
+        int ne13,
+        ulong nb11,
+        ulong nb12,
+        ulong nb13,
+        int ne20,
+        int ne21,
+        int ne22,
+        int ne23,
+        ulong nb21,
+        ulong nb22,
+        ulong nb23,
+        ulong nb1,
+        ulong nb2,
+        ulong nb3,
+        float eps,
+        local float * sum
+) {
+    src0 = src0 + offset0;
+    src1 = src1 + offset1;
+    src2 = src2 + offset2;
+    src3 = src3 + offset3;
+    dst  = dst  + offsetd;
+
+    const float sc = *((global float *) src3);
+
+    if (get_sub_group_id() == 0) {
+        sum[get_sub_group_local_id()] = 0.0f;
+    }
+
+    int i03 = get_group_id(2);
+    int i02 = get_group_id(1);
+    int i01 = get_group_id(0);
+
+    global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);
+    global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11);
+    global float4 * g = (global float4 *) (src2 + (i03%ne23)*nb23 + (i02%ne22)*nb22 + (i01%ne21)*nb21);
+
+    float sumf = 0;
+
+    for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
+        sumf += dot(x[i00], x[i00]);
+    }
+    sumf = sub_group_reduce_add(sumf);
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (get_sub_group_local_id() == 0) {
+        sum[get_sub_group_id()] = sumf;
+    }
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    sumf = sum[get_sub_group_local_id()];
+    sumf = sub_group_reduce_add(sumf);
+
+    float mean  = sumf / ne00;
+    float scale = 1.0f/sqrt(mean + eps);
+
+    global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1);
+    for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
+        y[i00] = ((x[i00] * scale) * f[i00%(ne10/4)] + g[i00%(ne20/4)]) * sc;
+    }
+}
@@ -0,0 +1,871 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load : enable
+#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load : enable
+
+#define bool2 uchar2
+#define bool3 uchar3
+#define bool4 uchar4
+
+__constant sampler_t smp_none = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_NONE | CLK_FILTER_NEAREST;
+__constant sampler_t smp_zero = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;
+
+__kernel void adreno_xmem_attn_q_f32_to_img_scaled(const global void *  src_void,
+                                                   ulong                src_offset,
+                                                   write_only image2d_t dst_image2d,
+                                                   const float          scale,
+                                                   const int            d_head,
+                                                   const int            n_q,
+                                                   const int            n_head,
+                                                   const int            n_head_kv,
+                                                   const int            n_batch,
+                                                   const ulong          src_nb1,
+                                                   const ulong          src_nb2,
+                                                   const ulong          src_nb3) {
+    const int x      = get_global_id(0);
+    const int flat_h = get_global_id(1);
+    const int d      = get_global_id(2);
+
+    const int heads_total = n_head * n_batch;
+    const int kpack       = d_head / 4;
+
+    if (x >= n_q || flat_h >= heads_total || d >= kpack) {
+        return;
+    }
+
+    const int batch = flat_h / n_head;
+    const int head  = flat_h % n_head;
+    const int gqa   = n_head / n_head_kv;
+    const int head_kv = head / gqa;
+    const int head_group = head - head_kv * gqa;
+    const int compact_h = batch * n_head_kv + head_kv;
+    const int compact_x = head_group * n_q + x;
+    const int c     = d * 4;
+
+    const global char *  src_base = (const global char *) src_void + src_offset;
+    const global float * row_ptr  = (const global float *) (src_base + batch * src_nb3 + head * src_nb2 + x * src_nb1);
+
+    half4 out = (half4) (0.0h);
+    out.x     = convert_half(row_ptr[c + 0] * scale);
+    if (c + 1 < d_head) {
+        out.y = convert_half(row_ptr[c + 1] * scale);
+    }
+    if (c + 2 < d_head) {
+        out.z = convert_half(row_ptr[c + 2] * scale);
+    }
+    if (c + 3 < d_head) {
+        out.w = convert_half(row_ptr[c + 3] * scale);
+    }
+
+    write_imageh(dst_image2d, (int2) (compact_x, compact_h * kpack + d), out);
+}
+
+__kernel void adreno_xmem_attn_kv_f32_to_img_gqa(const global void *  src_void,
+                                                 ulong                src_offset,
+                                                 write_only image2d_t dst_image2d,
+                                                 const int            d_head,
+                                                 const int            n_kv,
+                                                 const int            n_kv_padded,
+                                                 const int            n_head_kv,
+                                                 const int            n_batch,
+                                                 const ulong          src_nb1,
+                                                 const ulong          src_nb2,
+                                                 const ulong          src_nb3) {
+    const int x      = get_global_id(0);
+    const int flat_h = get_global_id(1);
+    const int d      = get_global_id(2);
+
+    const int kv_heads_total = n_head_kv * n_batch;
+    const int kpack          = d_head / 4;
+
+    if (x >= n_kv_padded || flat_h >= kv_heads_total || d >= kpack) {
+        return;
+    }
+
+    const int batch   = flat_h / n_head_kv;
+    const int head_kv = flat_h % n_head_kv;
+    const int c       = d * 4;
+
+    half4 out = (half4) (0.0h);
+    if (x < n_kv) {
+        const global char *  src_base = (const global char *) src_void + src_offset;
+        const global float * row_ptr =
+            (const global float *) (src_base + batch * src_nb3 + head_kv * src_nb2 + x * src_nb1);
+        out.x = convert_half(row_ptr[c + 0]);
+        if (c + 1 < d_head) {
+            out.y = convert_half(row_ptr[c + 1]);
+        }
+        if (c + 2 < d_head) {
+            out.z = convert_half(row_ptr[c + 2]);
+        }
+        if (c + 3 < d_head) {
+            out.w = convert_half(row_ptr[c + 3]);
+        }
+    }
+
+    write_imageh(dst_image2d, (int2) (x, flat_h * kpack + d), out);
+}
+
+__kernel void adreno_xmem_attn_kv_f16_to_img_gqa(const global void *  src_void,
+                                                 ulong                src_offset,
+                                                 write_only image2d_t dst_image2d,
+                                                 const int            d_head,
+                                                 const int            n_kv,
+                                                 const int            n_kv_padded,
+                                                 const int            n_head_kv,
+                                                 const int            n_batch,
+                                                 const ulong          src_nb1,
+                                                 const ulong          src_nb2,
+                                                 const ulong          src_nb3) {
+    const int x      = get_global_id(0);
+    const int flat_h = get_global_id(1);
+    const int d      = get_global_id(2);
+
+    const int kv_heads_total = n_head_kv * n_batch;
+    const int kpack          = d_head / 4;
+
+    if (x >= n_kv_padded || flat_h >= kv_heads_total || d >= kpack) {
+        return;
+    }
+
+    const int batch   = flat_h / n_head_kv;
+    const int head_kv = flat_h % n_head_kv;
+    const int c       = d * 4;
+
+    half4 out = (half4) (0.0h);
+    if (x < n_kv) {
+        const global char * src_base = (const global char *) src_void + src_offset;
+        const global half * row_ptr =
+            (const global half *) (src_base + batch * src_nb3 + head_kv * src_nb2 + x * src_nb1);
+        out.x = row_ptr[c + 0];
+        if (c + 1 < d_head) {
+            out.y = row_ptr[c + 1];
+        }
+        if (c + 2 < d_head) {
+            out.z = row_ptr[c + 2];
+        }
+        if (c + 3 < d_head) {
+            out.w = row_ptr[c + 3];
+        }
+    }
+
+    write_imageh(dst_image2d, (int2) (x, flat_h * kpack + d), out);
+}
+
+__kernel void adreno_xmem_attn_img_to_f32(global void *       dst_void,
+                                          ulong               dst_offset,
+                                          read_only image2d_t src_image2d,
+                                          const int           d_head,
+                                          const int           n_q,
+                                          const int           n_head,
+                                          const int           n_head_kv,
+                                          const int           n_batch,
+                                          const ulong         dst_nb1,
+                                          const ulong         dst_nb2,
+                                          const ulong         dst_nb3) {
+    const int x      = get_global_id(0);
+    const int flat_h = get_global_id(1);
+    const int d      = get_global_id(2);
+
+    const int heads_total = n_head * n_batch;
+    const int kpack       = d_head / 4;
+
+    if (x >= n_q || flat_h >= heads_total || d >= kpack) {
+        return;
+    }
+
+    const int batch = flat_h / n_head;
+    const int head  = flat_h % n_head;
+    const int gqa   = n_head / n_head_kv;
+    const int head_kv = head / gqa;
+    const int head_group = head - head_kv * gqa;
+    const int compact_h = batch * n_head_kv + head_kv;
+    const int compact_x = head_group * n_q + x;
+    const int c     = d * 4;
+
+    global char *  dst_base = (global char *) dst_void + dst_offset;
+    global float * row_ptr  = (global float *) (dst_base + batch * dst_nb3 + x * dst_nb2 + head * dst_nb1);
+
+    const half4 in_value = read_imageh(src_image2d, smp_zero, (int2) (compact_x, compact_h * kpack + d));
+    row_ptr[c + 0]       = convert_float(in_value.x);
+    if (c + 1 < d_head) {
+        row_ptr[c + 1] = convert_float(in_value.y);
+    }
+    if (c + 2 < d_head) {
+        row_ptr[c + 2] = convert_float(in_value.z);
+    }
+    if (c + 3 < d_head) {
+        row_ptr[c + 3] = convert_float(in_value.w);
+    }
+}
+
+__kernel void adreno_xmem_attn_k_gather(global half4 *      dst_tensor_buffer,
+                                        read_only image2d_t src_tensor_image2d,
+                                        const int4          shared_int4_0,
+                                        const int4          shared_int4_1) {
+    int X = get_global_id(0);
+    int Y = get_global_id(1);
+    int S = get_global_id(2);
+    if (X >= shared_int4_0.w || Y >= shared_int4_0.y || S >= shared_int4_0.z) {
+        return;
+    }
+    half temps[4];
+    temps[0] = (half) (0.f);
+    temps[1] = (half) (0.f);
+    temps[2] = (half) (0.f);
+    temps[3] = (half) (0.f);
+    for (int i = 0; i < 4; ++i) {
+        int dst_channel = S * 4 + i;
+        if (dst_channel < shared_int4_0.x) {
+            int s_y = Y;
+            int s_x = dst_channel;
+            int s_c = X;
+            {
+                int   slice_coord_TMP  = (s_c) / 4;
+                int   sub_ch_coord_TMP = (s_c) % 4;
+                half4 src_TMP          = read_imageh(src_tensor_image2d, smp_zero,
+                                                     (int2) ((s_x), ((s_y) *shared_int4_1.x + (slice_coord_TMP))));
+                temps[i]               = (half[4]){ src_TMP.x, src_TMP.y, src_TMP.z, src_TMP.w }[sub_ch_coord_TMP];
+            };
+        }
+    }
+    half4 result;
+    result.x                                                                  = temps[0];
+    result.y                                                                  = temps[1];
+    result.z                                                                  = temps[2];
+    result.w                                                                  = temps[3];
+    dst_tensor_buffer[(((S) *shared_int4_0.y + (Y)) * shared_int4_0.w + (X))] = result;
+}
+
+__kernel void adreno_xmem_attn_pack_k(global half4 *             dst_tensor_buffer,
+                                      read_only image1d_buffer_t src_image_buffer,
+                                      const int4                 shared_int4_0,
+                                      const int4                 shared_int4_1,
+                                      const int4                 shared_int4_2) {
+    int linear_index = get_global_id(0);
+    if (linear_index >= shared_int4_0.y) {
+        return;
+    }
+    if (get_global_id(1) != 0) {
+        return;
+    }
+    if (get_global_id(2) != 0) {
+        return;
+    }
+    int   dst_o_sp_i_ogroup = linear_index;
+    int   dst_ogroup        = dst_o_sp_i_ogroup % shared_int4_0.x;
+    int   dst_o_sp_i        = dst_o_sp_i_ogroup / shared_int4_0.x;
+    int   dst_i             = dst_o_sp_i % shared_int4_0.z;
+    int   dst_o_sp          = dst_o_sp_i / shared_int4_0.z;
+    int   dst_sp            = dst_o_sp % shared_int4_1.x;
+    int   dst_o             = dst_o_sp / shared_int4_1.x;
+    int   i_slice           = dst_i;
+    int   o_slice           = dst_o * shared_int4_0.x + dst_ogroup;
+    int   spatial_linear    = dst_sp;
+    int   W                 = spatial_linear % shared_int4_1.y;
+    int   H                 = spatial_linear / shared_int4_1.y;
+    half4 w0                = (half4) (0);
+    half4 w1                = (half4) (0);
+    half4 w2                = (half4) (0);
+    half4 w3                = (half4) (0);
+
+    if (i_slice * 4 < shared_int4_0.w && o_slice < shared_int4_1.w) {
+        w0 = read_imageh(src_image_buffer, (((o_slice) *shared_int4_1.z + (W)) * shared_int4_2.x + (i_slice * 4)));
+    }
+    if (i_slice * 4 + 1 < shared_int4_0.w && o_slice < shared_int4_1.w) {
+        w1 = read_imageh(src_image_buffer, (((o_slice) *shared_int4_1.z + (W)) * shared_int4_2.x + (i_slice * 4 + 1)));
+    }
+    if (i_slice * 4 + 2 < shared_int4_0.w && o_slice < shared_int4_1.w) {
+        w2 = read_imageh(src_image_buffer, (((o_slice) *shared_int4_1.z + (W)) * shared_int4_2.x + (i_slice * 4 + 2)));
+    }
+    if (i_slice * 4 + 3 < shared_int4_0.w && o_slice < shared_int4_1.w) {
+        w3 = read_imageh(src_image_buffer, (((o_slice) *shared_int4_1.z + (W)) * shared_int4_2.x + (i_slice * 4 + 3)));
+    }
+    half4 r0                                = w0;
+    half4 r1                                = w1;
+    half4 r2                                = w2;
+    half4 r3                                = w3;
+    dst_tensor_buffer[linear_index * 4 + 0] = r0;
+    dst_tensor_buffer[linear_index * 4 + 1] = r1;
+    dst_tensor_buffer[linear_index * 4 + 2] = r2;
+    dst_tensor_buffer[linear_index * 4 + 3] = r3;
+}
+
+__attribute__((qcom_max_concurrent_subgroups(12))) __kernel void adreno_xmem_attn_qk_gemm(
+    global half4 *      dst_tensor_buffer,
+    constant half8 *    weights_buffer __attribute__((sub_group_uniform)),
+    constant half8 *    xmem_buffer __attribute__((max_constant_size((6144)))),
+    read_only image2d_t src_tensor_image2d,
+    const int4          shared_int4_0,
+    const int4          shared_int4_1,
+    const int4          shared_int4_2) {
+    int X = get_group_id(1) * get_local_size(0) + get_local_id(0);
+    int Y = get_group_id(2) * get_local_size(1) + get_local_id(1);
+    int Z = get_group_id(0) * get_local_size(2) + get_local_id(2);
+    if (X >= shared_int4_0.z || Y >= shared_int4_0.x) {
+        return;
+    }
+    if (Z * 8 >= shared_int4_0.y) {
+        return;
+    }
+
+    half4 r0      = (half4) (0.f);
+    half4 r1      = (half4) (0.f);
+    half4 r2      = (half4) (0.f);
+    half4 r3      = (half4) (0.f);
+    half4 r4      = (half4) (0.f);
+    half4 r5      = (half4) (0.f);
+    half4 r6      = (half4) (0.f);
+    half4 r7      = (half4) (0.f);
+    int   x_coord = mad24(X, shared_int4_2.y, shared_int4_1.y);
+    int   y_coord = mad24(Y, shared_int4_2.z, shared_int4_1.z);
+    int   coord_x, coord_y, coord_s;
+    int   f_offset = (Z * shared_int4_1.w + Y) * shared_int4_1.x * 32;
+
+    int subgroup_id                   = (int) ((0x1F & qcom_get_physical_sub_group_id()));
+    subgroup_id                       = subgroup_id % 12;
+    int                 c_offset      = mul24(subgroup_id, shared_int4_0.w);
+    __constant half16 * weights_cache = (__constant half16 *) &xmem_buffer[c_offset];
+    coord_y                           = Y;
+    coord_x                           = X;
+    coord_s                           = 0;
+    do {
+        half4 src0 =
+            read_imageh(src_tensor_image2d, smp_zero, (int2) ((coord_x), ((coord_y) *shared_int4_2.x + (coord_s))));
+        coord_s++;
+        half4 src1 =
+            read_imageh(src_tensor_image2d, smp_zero, (int2) ((coord_x), ((coord_y) *shared_int4_2.x + (coord_s))));
+        coord_s++;
+        qcom_sub_group_constant_load8(xmem_buffer, weights_buffer, c_offset, f_offset >> 1, 32);
+        f_offset += 64;
+        qcom_sub_group_sync(QCOM_CLK_CONST_LOAD_SYNC);
+        r0 += src0.x * weights_cache[0].s0123;
+        r0 += src0.y * weights_cache[0].s4567;
+        r0 += src0.z * weights_cache[0].s89ab;
+        r0 += src0.w * weights_cache[0].scdef;
+        r1 += src0.x * weights_cache[1].s0123;
+        r1 += src0.y * weights_cache[1].s4567;
+        r1 += src0.z * weights_cache[1].s89ab;
+        r1 += src0.w * weights_cache[1].scdef;
+        r2 += src0.x * weights_cache[2].s0123;
+        r2 += src0.y * weights_cache[2].s4567;
+        r2 += src0.z * weights_cache[2].s89ab;
+        r2 += src0.w * weights_cache[2].scdef;
+        r3 += src0.x * weights_cache[3].s0123;
+        r3 += src0.y * weights_cache[3].s4567;
+        r3 += src0.z * weights_cache[3].s89ab;
+        r3 += src0.w * weights_cache[3].scdef;
+        r4 += src0.x * weights_cache[4].s0123;
+        r4 += src0.y * weights_cache[4].s4567;
+        r4 += src0.z * weights_cache[4].s89ab;
+        r4 += src0.w * weights_cache[4].scdef;
+        r5 += src0.x * weights_cache[5].s0123;
+        r5 += src0.y * weights_cache[5].s4567;
+        r5 += src0.z * weights_cache[5].s89ab;
+        r5 += src0.w * weights_cache[5].scdef;
+        r6 += src0.x * weights_cache[6].s0123;
+        r6 += src0.y * weights_cache[6].s4567;
+        r6 += src0.z * weights_cache[6].s89ab;
+        r6 += src0.w * weights_cache[6].scdef;
+        r7 += src0.x * weights_cache[7].s0123;
+        r7 += src0.y * weights_cache[7].s4567;
+        r7 += src0.z * weights_cache[7].s89ab;
+        r7 += src0.w * weights_cache[7].scdef;
+        r0 += src1.x * weights_cache[8].s0123;
+        r0 += src1.y * weights_cache[8].s4567;
+        r0 += src1.z * weights_cache[8].s89ab;
+        r0 += src1.w * weights_cache[8].scdef;
+        r1 += src1.x * weights_cache[9].s0123;
+        r1 += src1.y * weights_cache[9].s4567;
+        r1 += src1.z * weights_cache[9].s89ab;
+        r1 += src1.w * weights_cache[9].scdef;
+        r2 += src1.x * weights_cache[10].s0123;
+        r2 += src1.y * weights_cache[10].s4567;
+        r2 += src1.z * weights_cache[10].s89ab;
+        r2 += src1.w * weights_cache[10].scdef;
+        r3 += src1.x * weights_cache[11].s0123;
+        r3 += src1.y * weights_cache[11].s4567;
+        r3 += src1.z * weights_cache[11].s89ab;
+        r3 += src1.w * weights_cache[11].scdef;
+        r4 += src1.x * weights_cache[12].s0123;
+        r4 += src1.y * weights_cache[12].s4567;
+        r4 += src1.z * weights_cache[12].s89ab;
+        r4 += src1.w * weights_cache[12].scdef;
+        r5 += src1.x * weights_cache[13].s0123;
+        r5 += src1.y * weights_cache[13].s4567;
+        r5 += src1.z * weights_cache[13].s89ab;
+        r5 += src1.w * weights_cache[13].scdef;
+        r6 += src1.x * weights_cache[14].s0123;
+        r6 += src1.y * weights_cache[14].s4567;
+        r6 += src1.z * weights_cache[14].s89ab;
+        r6 += src1.w * weights_cache[14].scdef;
+        r7 += src1.x * weights_cache[15].s0123;
+        r7 += src1.y * weights_cache[15].s4567;
+        r7 += src1.z * weights_cache[15].s89ab;
+        r7 += src1.w * weights_cache[15].scdef;
+    } while (coord_s < shared_int4_2.x);
+
+    coord_s = mul24(Z, 8);
+    coord_x = X;
+    coord_y = Y;
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r0);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0))));
+        }
+        dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res;
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r1);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0))));
+        }
+        dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res;
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r2);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0))));
+        }
+        dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res;
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r3);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0))));
+        }
+        dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res;
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r4);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0))));
+        }
+        dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res;
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r5);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0))));
+        }
+        dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res;
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r6);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0))));
+        }
+        dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res;
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r7);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0))));
+        }
+        dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res;
+        coord_s++;
+    }
+}
+
+__kernel void adreno_xmem_attn_softmax_reduce_basic(read_only image1d_buffer_t src_tensor_image_buffer,
+                                                    write_only image2d_t       dst_tensor_image2d,
+                                                    const int4                 shared_int4_0,
+                                                    const int4                 shared_int4_1) {
+    int X = get_global_id(0);
+    int Y = get_global_id(1);
+    if (X >= shared_int4_0.z || Y >= shared_int4_0.x) {
+        return;
+    }
+    float sum                     = 0.0f;
+    int   end_channel             = shared_int4_0.w;
+    int   end_slice               = (end_channel + 3) / 4;
+    int   start_channel           = 0;
+    int   start_slice             = start_channel / 4;
+    bool  need_per_channels_check = start_channel % 4 != 0 || end_channel % 4 != 0;
+    float maximum;
+    {
+        int    slice_coord_TMP  = (start_channel) / 4;
+        int    sub_ch_coord_TMP = (start_channel) % 4;
+        float4 src_TMP          = convert_float4(
+            read_imageh(src_tensor_image_buffer, ((slice_coord_TMP) *shared_int4_1.x + (Y)) * shared_int4_1.y + (X)));
+        maximum = (float[4]){ src_TMP.x, src_TMP.y, src_TMP.z, src_TMP.w }[sub_ch_coord_TMP];
+    };
+    for (int d = start_slice; d < end_slice; d += 1) {
+        float4 mask_dot = (float4) (1.f);
+        float4 src =
+            convert_float4(read_imageh(src_tensor_image_buffer, ((d) *shared_int4_1.x + (Y)) * shared_int4_1.y + (X)));
+        if (need_per_channels_check && (d == start_slice || d == end_slice - 1)) {
+            if (d * 4 + 0 < start_channel || d * 4 + 0 >= end_channel) {
+                mask_dot.x = 0.f;
+                src.x      = maximum;
+            }
+            if (d * 4 + 1 < start_channel || d * 4 + 1 >= end_channel) {
+                mask_dot.y = 0.f;
+                src.y      = maximum;
+            }
+            if (d * 4 + 2 < start_channel || d * 4 + 2 >= end_channel) {
+                mask_dot.z = 0.f;
+                src.z      = maximum;
+            }
+            if (d * 4 + 3 < start_channel || d * 4 + 3 >= end_channel) {
+                mask_dot.w = 0.f;
+                src.w      = maximum;
+            }
+        }
+        float new_max = max(src.x, src.y);
+        new_max       = max(new_max, src.z);
+        new_max       = max(new_max, src.w);
+        new_max       = max(new_max, maximum);
+        float scale   = native_exp(maximum - new_max);
+        maximum       = new_max;
+        sum *= scale;
+        float4 exp_res = native_exp(src - maximum);
+        sum += dot(mask_dot, exp_res);
+    }
+    if (!isfinite(maximum) || sum == 0.0f) {
+        write_imageh(dst_tensor_image2d, (int2) (X, Y), (half4) (0.0h));
+        return;
+    }
+    write_imageh(dst_tensor_image2d, (int2) (X, Y),
+                 (half4) (convert_half(1.0f / sum), convert_half(maximum), 0.0h, 0.0h));
+}
+
+__kernel void adreno_xmem_attn_softmax_apply_basic(global half4 *             dst_tensor_buffer,
+                                                   read_only image1d_buffer_t src_tensor_image_buffer,
+                                                   read_only image2d_t        src_tensor_1_image2d,
+                                                   const int4                 shared_int4_0,
+                                                   const int4                 shared_int4_1) {
+    int X = get_global_id(0);
+    int Y = get_global_id(1);
+    int Z = get_global_id(2);
+    if (X >= shared_int4_0.z || Y >= shared_int4_0.x || Z >= shared_int4_0.y) {
+        return;
+    }
+    half4 src = read_imageh(src_tensor_image_buffer, ((Z) *shared_int4_1.x + (Y)) * shared_int4_1.y + (X));
+    {
+        half4 src_final;
+        {
+            {
+                half4 exp_val = read_imageh(src_tensor_1_image2d, smp_zero, (int2) (X, Y));
+                src_final = exp(src - exp_val.y) * exp_val.x;
+                const int k = Z * 4;
+                const int n_kv = shared_int4_1.z;
+                if (k + 0 >= n_kv) {
+                    src_final.x = 0.0h;
+                }
+                if (k + 1 >= n_kv) {
+                    src_final.y = 0.0h;
+                }
+                if (k + 2 >= n_kv) {
+                    src_final.z = 0.0h;
+                }
+                if (k + 3 >= n_kv) {
+                    src_final.w = 0.0h;
+                }
+            }
+        }
+        dst_tensor_buffer[(((Z) *shared_int4_0.x + (Y)) * shared_int4_0.z + (X))] = src_final;
+    };
+}
+
+__kernel void adreno_xmem_attn_mask_scores(global half4 *             dst_score_tensor_buffer,
+                                           read_only image1d_buffer_t src_score_image_buffer,
+                                           const global half *        mask,
+                                           const ulong                mask_offset,
+                                           const int                  q_width,
+                                           const int                  n_q,
+                                           const int                  n_kv,
+                                           const int                  n_kv_padded,
+                                           const int                  kv_heads_total,
+                                           const int                  n_head,
+                                           const int                  n_head_kv,
+                                           const ulong                mask_nb1,
+                                           const ulong                mask_nb2,
+                                           const ulong                mask_nb3,
+                                           const int                  mask_ne2,
+                                           const int                  mask_ne3) {
+    const int X     = get_global_id(0);
+    const int Y     = get_global_id(1);
+    const int Z     = get_global_id(2);
+    const int npack = n_kv_padded / 4;
+    if (X >= q_width || Y >= kv_heads_total || Z >= npack) {
+        return;
+    }
+
+    const int           gqa            = n_head / n_head_kv;
+    const int           head_kv        = Y % n_head_kv;
+    const int           batch          = Y / n_head_kv;
+    const int           head_group     = X / n_q;
+    const int           q              = X - head_group * n_q;
+    const int           head           = head_kv * gqa + head_group;
+    const int           mask_head_idx  = head % mask_ne2;
+    const int           mask_batch_idx = batch % mask_ne3;
+    const global char *  mask_base      = (const global char *) mask + mask_offset;
+    const global half *  mask_row       = (const global half *) (mask_base + mask_batch_idx * mask_nb3 +
+                                                                 mask_head_idx * mask_nb2 + q * mask_nb1);
+
+    const half4 score   = read_imageh(src_score_image_buffer, ((Z * kv_heads_total + Y) * q_width + X));
+    float       vals[4] = {
+        convert_float(score.x),
+        convert_float(score.y),
+        convert_float(score.z),
+        convert_float(score.w),
+    };
+
+    for (int lane = 0; lane < 4; ++lane) {
+        const int k_idx = Z * 4 + lane;
+        if (k_idx >= n_kv) {
+            vals[lane] = -INFINITY;
+        } else {
+            vals[lane] += convert_float(mask_row[k_idx]);
+        }
+    }
+
+    dst_score_tensor_buffer[((Z * kv_heads_total + Y) * q_width + X)] =
+        (half4) (convert_half(vals[0]), convert_half(vals[1]), convert_half(vals[2]), convert_half(vals[3]));
+}
+
+__kernel void adreno_xmem_attn_pack_v(global half4 *      dst_tensor_buffer,
+                                      read_only image2d_t src_image2d,
+                                      const int4          shared_int4_0,
+                                      const int4          shared_int4_1) {
+    int linear_index = get_global_id(0);
+    if (linear_index >= shared_int4_0.y) {
+        return;
+    }
+    if (get_global_id(1) != 0) {
+        return;
+    }
+    if (get_global_id(2) != 0) {
+        return;
+    }
+    int   dst_o_sp_i_ogroup = linear_index;
+    int   dst_ogroup        = dst_o_sp_i_ogroup % shared_int4_0.x;
+    int   dst_o_sp_i        = dst_o_sp_i_ogroup / shared_int4_0.x;
+    int   dst_i             = dst_o_sp_i % shared_int4_0.z;
+    int   dst_o_sp          = dst_o_sp_i / shared_int4_0.z;
+    int   dst_sp            = dst_o_sp % shared_int4_1.x;
+    int   dst_o             = dst_o_sp / shared_int4_1.x;
+    int   i_slice           = dst_i;
+    int   o_slice           = dst_o * shared_int4_0.x + dst_ogroup;
+    int   spatial_linear    = dst_sp;
+    int   W                 = spatial_linear % shared_int4_1.y;
+    int   H                 = spatial_linear / shared_int4_1.y;
+    half4 w0                = (half4) (0);
+    half4 w1                = (half4) (0);
+    half4 w2                = (half4) (0);
+    half4 w3                = (half4) (0);
+
+    if (i_slice * 4 < shared_int4_0.w && o_slice < shared_int4_1.z) {
+        w0 = read_imageh(src_image2d, smp_zero, (int2) ((i_slice * 4), ((W) *shared_int4_1.z + (o_slice))));
+    }
+    if (i_slice * 4 + 1 < shared_int4_0.w && o_slice < shared_int4_1.z) {
+        w1 = read_imageh(src_image2d, smp_zero, (int2) ((i_slice * 4 + 1), ((W) *shared_int4_1.z + (o_slice))));
+    }
+    if (i_slice * 4 + 2 < shared_int4_0.w && o_slice < shared_int4_1.z) {
+        w2 = read_imageh(src_image2d, smp_zero, (int2) ((i_slice * 4 + 2), ((W) *shared_int4_1.z + (o_slice))));
+    }
+    if (i_slice * 4 + 3 < shared_int4_0.w && o_slice < shared_int4_1.z) {
+        w3 = read_imageh(src_image2d, smp_zero, (int2) ((i_slice * 4 + 3), ((W) *shared_int4_1.z + (o_slice))));
+    }
+    half4 r0                                = w0;
+    half4 r1                                = w1;
+    half4 r2                                = w2;
+    half4 r3                                = w3;
+    dst_tensor_buffer[linear_index * 4 + 0] = r0;
+    dst_tensor_buffer[linear_index * 4 + 1] = r1;
+    dst_tensor_buffer[linear_index * 4 + 2] = r2;
+    dst_tensor_buffer[linear_index * 4 + 3] = r3;
+}
+
+__attribute__((qcom_max_concurrent_subgroups(12))) __kernel void adreno_xmem_attn_pv_gemm(
+    constant half8 *           weights_buffer __attribute__((sub_group_uniform)),
+    constant half8 *           xmem_buffer __attribute__((max_constant_size((6144)))),
+    read_only image1d_buffer_t src_tensor_image_buffer,
+    write_only image2d_t       dst_tensor_image2d,
+    const int4                 shared_int4_0,
+    const int4                 shared_int4_1,
+    const int4                 shared_int4_2,
+    const int4                 shared_int4_3) {
+    int X = get_group_id(1) * get_local_size(0) + get_local_id(0);
+    int Y = get_group_id(2) * get_local_size(1) + get_local_id(1);
+    int Z = get_group_id(0) * get_local_size(2) + get_local_id(2);
+    if (X >= shared_int4_0.z || Y >= shared_int4_0.x) {
+        return;
+    }
+    if (Z * 8 >= shared_int4_0.y) {
+        return;
+    }
+
+    half4 r0      = (half4) (0.f);
+    half4 r1      = (half4) (0.f);
+    half4 r2      = (half4) (0.f);
+    half4 r3      = (half4) (0.f);
+    half4 r4      = (half4) (0.f);
+    half4 r5      = (half4) (0.f);
+    half4 r6      = (half4) (0.f);
+    half4 r7      = (half4) (0.f);
+    int   x_coord = mad24(X, shared_int4_2.w, shared_int4_1.y);
+    int   y_coord = mad24(Y, shared_int4_3.x, shared_int4_1.z);
+    int   coord_x, coord_y, coord_s;
+    int   f_offset = (Z * shared_int4_1.w + Y) * shared_int4_1.x * 32;
+
+    int subgroup_id                   = (int) ((0x1F & qcom_get_physical_sub_group_id()));
+    subgroup_id                       = subgroup_id % 12;
+    int                 c_offset      = mul24(subgroup_id, shared_int4_0.w);
+    __constant half16 * weights_cache = (__constant half16 *) &xmem_buffer[c_offset];
+    coord_y                           = Y;
+    coord_x                           = X;
+    int addr                          = (((0) * shared_int4_1.w + (coord_y)) * shared_int4_2.z + (coord_x));
+    int dz                            = shared_int4_2.x;
+    coord_s                           = 0;
+    do {
+        half4 src0 = read_imageh(src_tensor_image_buffer, addr);
+        addr += dz;
+        coord_s++;
+        half4 src1 = read_imageh(src_tensor_image_buffer, addr);
+        addr += dz;
+        coord_s++;
+        qcom_sub_group_constant_load8(xmem_buffer, weights_buffer, c_offset, f_offset >> 1, 32);
+        f_offset += 64;
+        qcom_sub_group_sync(QCOM_CLK_CONST_LOAD_SYNC);
+        r0 += src0.x * weights_cache[0].s0123;
+        r0 += src0.y * weights_cache[0].s4567;
+        r0 += src0.z * weights_cache[0].s89ab;
+        r0 += src0.w * weights_cache[0].scdef;
+        r1 += src0.x * weights_cache[1].s0123;
+        r1 += src0.y * weights_cache[1].s4567;
+        r1 += src0.z * weights_cache[1].s89ab;
+        r1 += src0.w * weights_cache[1].scdef;
+        r2 += src0.x * weights_cache[2].s0123;
+        r2 += src0.y * weights_cache[2].s4567;
+        r2 += src0.z * weights_cache[2].s89ab;
+        r2 += src0.w * weights_cache[2].scdef;
+        r3 += src0.x * weights_cache[3].s0123;
+        r3 += src0.y * weights_cache[3].s4567;
+        r3 += src0.z * weights_cache[3].s89ab;
+        r3 += src0.w * weights_cache[3].scdef;
+        r4 += src0.x * weights_cache[4].s0123;
+        r4 += src0.y * weights_cache[4].s4567;
+        r4 += src0.z * weights_cache[4].s89ab;
+        r4 += src0.w * weights_cache[4].scdef;
+        r5 += src0.x * weights_cache[5].s0123;
+        r5 += src0.y * weights_cache[5].s4567;
+        r5 += src0.z * weights_cache[5].s89ab;
+        r5 += src0.w * weights_cache[5].scdef;
+        r6 += src0.x * weights_cache[6].s0123;
+        r6 += src0.y * weights_cache[6].s4567;
+        r6 += src0.z * weights_cache[6].s89ab;
+        r6 += src0.w * weights_cache[6].scdef;
+        r7 += src0.x * weights_cache[7].s0123;
+        r7 += src0.y * weights_cache[7].s4567;
+        r7 += src0.z * weights_cache[7].s89ab;
+        r7 += src0.w * weights_cache[7].scdef;
+        r0 += src1.x * weights_cache[8].s0123;
+        r0 += src1.y * weights_cache[8].s4567;
+        r0 += src1.z * weights_cache[8].s89ab;
+        r0 += src1.w * weights_cache[8].scdef;
+        r1 += src1.x * weights_cache[9].s0123;
+        r1 += src1.y * weights_cache[9].s4567;
+        r1 += src1.z * weights_cache[9].s89ab;
+        r1 += src1.w * weights_cache[9].scdef;
+        r2 += src1.x * weights_cache[10].s0123;
+        r2 += src1.y * weights_cache[10].s4567;
+        r2 += src1.z * weights_cache[10].s89ab;
+        r2 += src1.w * weights_cache[10].scdef;
+        r3 += src1.x * weights_cache[11].s0123;
+        r3 += src1.y * weights_cache[11].s4567;
+        r3 += src1.z * weights_cache[11].s89ab;
+        r3 += src1.w * weights_cache[11].scdef;
+        r4 += src1.x * weights_cache[12].s0123;
+        r4 += src1.y * weights_cache[12].s4567;
+        r4 += src1.z * weights_cache[12].s89ab;
+        r4 += src1.w * weights_cache[12].scdef;
+        r5 += src1.x * weights_cache[13].s0123;
+        r5 += src1.y * weights_cache[13].s4567;
+        r5 += src1.z * weights_cache[13].s89ab;
+        r5 += src1.w * weights_cache[13].scdef;
+        r6 += src1.x * weights_cache[14].s0123;
+        r6 += src1.y * weights_cache[14].s4567;
+        r6 += src1.z * weights_cache[14].s89ab;
+        r6 += src1.w * weights_cache[14].scdef;
+        r7 += src1.x * weights_cache[15].s0123;
+        r7 += src1.y * weights_cache[15].s4567;
+        r7 += src1.z * weights_cache[15].s89ab;
+        r7 += src1.w * weights_cache[15].scdef;
+    } while (coord_s < shared_int4_2.y);
+
+    coord_s = mul24(Z, 8);
+    coord_x = X;
+    coord_y = Y;
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r0);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0));
+        }
+        write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res);
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r1);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0));
+        }
+        write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res);
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r2);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0));
+        }
+        write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res);
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r3);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0));
+        }
+        write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res);
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r4);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0));
+        }
+        write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res);
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r5);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0));
+        }
+        write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res);
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r6);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0));
+        }
+        write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res);
+        coord_s++;
+    }
+    if (coord_s < shared_int4_0.y) {
+        half4 res = convert_half4(r7);
+        if (coord_s < 0) {
+            res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0));
+        }
+        write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res);
+        coord_s++;
+    }
+}
@@ -0,0 +1,85 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+//------------------------------------------------------------------------------
+// Extended elementwise unary ops, same variant shape as abs.cl:
+//   f32, f32_4 (vec4), f16, f16_4 (vec4), f32_nc, f16_nc (stride-addressed).
+//
+//   sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc.
+//
+// Semantics match the ggml CPU reference (ggml.c). Values are computed in float
+// (the f16 variants read/write half and convert), so the conditional ops match
+// the CPU bit-for-bit within tolerance. SEXPR is the scalar form, VEXPR the
+// float4 form (vector ternaries need select()).
+//------------------------------------------------------------------------------
+
+#define UNARY_EXT(NAME, SEXPR, VEXPR)                                           \
+kernel void kernel_##NAME##_f32(                                               \
+        global const float * src0, ulong offset0,                             \
+        global       float * dst,  ulong offsetd) {                           \
+    src0 = (global float*)((global char*)src0 + offset0);                     \
+    dst  = (global float*)((global char*)dst  + offsetd);                     \
+    float x = src0[get_global_id(0)];                                         \
+    dst[get_global_id(0)] = (SEXPR);                                          \
+}                                                                             \
+kernel void kernel_##NAME##_f32_4(                                            \
+        global const float4 * src0, ulong offset0,                           \
+        global       float4 * dst,  ulong offsetd) {                         \
+    src0 = (global float4*)((global char*)src0 + offset0);                   \
+    dst  = (global float4*)((global char*)dst  + offsetd);                   \
+    float4 x = src0[get_global_id(0)];                                       \
+    dst[get_global_id(0)] = (VEXPR);                                         \
+}                                                                             \
+kernel void kernel_##NAME##_f16(                                             \
+        global const half * src0, ulong offset0,                            \
+        global       half * dst,  ulong offsetd) {                          \
+    src0 = (global half*)((global char*)src0 + offset0);                    \
+    dst  = (global half*)((global char*)dst  + offsetd);                    \
+    float x = src0[get_global_id(0)];                                       \
+    dst[get_global_id(0)] = (SEXPR);                                        \
+}                                                                            \
+kernel void kernel_##NAME##_f16_4(                                          \
+        global const half4 * src0, ulong offset0,                          \
+        global       half4 * dst,  ulong offsetd) {                        \
+    src0 = (global half4*)((global char*)src0 + offset0);                  \
+    dst  = (global half4*)((global char*)dst  + offsetd);                  \
+    float4 x = convert_float4(src0[get_global_id(0)]);                     \
+    dst[get_global_id(0)] = convert_half4(VEXPR);                          \
+}                                                                           \
+kernel void kernel_##NAME##_f32_nc(                                         \
+        global const char * src0, ulong offset0,                          \
+        global       char * dst,  ulong offsetd,                          \
+        int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03,         \
+        ulong nb0, ulong nb1, ulong nb2, ulong nb3) {                     \
+    src0 = src0 + offset0; dst = dst + offsetd;                            \
+    const int i3 = get_group_id(2);                                       \
+    const int i2 = get_group_id(1);                                       \
+    const int i1 = get_group_id(0);                                       \
+    for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {  \
+        float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \
+        *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR);            \
+    }                                                                     \
+}                                                                         \
+kernel void kernel_##NAME##_f16_nc(                                       \
+        global const char * src0, ulong offset0,                        \
+        global       char * dst,  ulong offsetd,                        \
+        int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03,       \
+        ulong nb0, ulong nb1, ulong nb2, ulong nb3) {                   \
+    src0 = src0 + offset0; dst = dst + offsetd;                          \
+    const int i3 = get_group_id(2);                                     \
+    const int i2 = get_group_id(1);                                     \
+    const int i1 = get_group_id(0);                                     \
+    for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\
+        float x = *(global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \
+        *(global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR);            \
+    }                                                                   \
+}
+
+UNARY_EXT(sgn,          sign(x),                                           sign(x))
+UNARY_EXT(step,         x > 0.0f ? 1.0f : 0.0f,                            select((float4)0.0f, (float4)1.0f, x > 0.0f))
+UNARY_EXT(elu,          x > 0.0f ? x : expm1(x),                           select(expm1(x), x, x > 0.0f))
+UNARY_EXT(hardswish,    x * fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)),     x * fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f)))
+UNARY_EXT(hardsigmoid,  fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)),         fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f)))
+UNARY_EXT(floor,        floor(x),                                          floor(x))
+UNARY_EXT(ceil,         ceil(x),                                           ceil(x))
+UNARY_EXT(round,        round(x),                                          round(x))
+UNARY_EXT(trunc,        trunc(x),                                          trunc(x))
@@ -1091,6 +1091,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
        if (op->ne[3] != 1) {
            return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
        }
+        if (op->op == GGML_OP_GET_ROWS && ggml_is_quantized(op->src[0]->type) &&
+            op->src[0]->view_src != nullptr && op->src[0]->view_offs != 0) {
+            return {false, "GET_ROWS with a nonzero quantized src0 view offset is not supported"};
+        }
        if (op->op == GGML_OP_GET_ROWS && ggml_openvino_get_device_name() == "GPU" &&
            op->src[0]->type == GGML_TYPE_BF16) {
            return {false, "GET_ROWS with BF16 src0 is not supported on GPU"};
@@ -835,7 +835,7 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty
    // [TAG_ALLOC_SIZE_EXPAND]
    // ops that may require additional memory for fleeting data on certain backends
    // ref: https://github.com/ggml-org/llama.cpp/pull/15966
-    rpc_get |= ggml_backend_op_alloc_size_may_expand(tensor->op);
+    rpc_get |= ggml_op_alloc_size_may_expand(tensor->op);

    if (rpc_get) {
        ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
@@ -1,5 +1,6 @@
 #include "binbcast.hpp"

+#include <algorithm>
 #include <cstddef>
 #include <cstdint>
 #include <sycl/sycl.hpp>
@@ -356,3 +357,294 @@ void ggml_sycl_repeat(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
    ggml_sycl_op_repeat(ctx, dst);
 }

+// fused ADD+ADD: dst = (src0 + src1) + src2. Same indexing as k_bin_bcast, so mixed
+// types, broadcast, and non-contiguous layouts that add() already handles also fuse.
+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename src2_t, typename dst_t>
+static void k_bin_bcast3(const src0_t * src0, const src1_t * src1, const src2_t * src2, dst_t * dst,
+        int ne0, int ne1, int ne2, int ne3,
+        int ne10, int ne11, int ne12, int ne13,
+        int ne20, int ne21, int ne22, int ne23,
+        int s1, int s2, int s3,
+        int s00, int s01, int s02, int s03,
+        int s10, int s11, int s12, int s13,
+        int s20, int s21, int s22, int s23,
+        const sycl::nd_item<3> & item_ct1) {
+    const int i0s = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
+                    item_ct1.get_local_id(2);
+    const int i1 = (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
+                    item_ct1.get_local_id(1));
+    const int i2 = (item_ct1.get_local_range(0) * item_ct1.get_group(0) +
+                    item_ct1.get_local_id(0)) /
+                   ne3;
+    const int i3 = (item_ct1.get_local_range(0) * item_ct1.get_group(0) +
+                    item_ct1.get_local_id(0)) %
+                   ne3;
+
+    if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
+        return;
+    }
+
+    const int i11 = i1 % ne11;
+    const int i12 = i2 % ne12;
+    const int i13 = i3 % ne13;
+    const int i21 = i1 % ne21;
+    const int i22 = i2 % ne22;
+    const int i23 = i3 % ne23;
+
+    const size_t i_src0 = i3 * s03 + i2 * s02 + i1 * s01;
+    const size_t i_src1 = i13 * s13 + i12 * s12 + i11 * s11;
+    const size_t i_src2 = i23 * s23 + i22 * s22 + i21 * s21;
+    const size_t i_dst  = i3 * s3 + i2 * s2 + i1 * s1;
+
+    const src0_t * src0_row = src0 + i_src0;
+    const src1_t * src1_row = src1 + i_src1;
+    const src2_t * src2_row = src2 + i_src2;
+    dst_t *        dst_row  = dst + i_dst;
+
+    for (int i0 = i0s; i0 < ne0;
+         i0 += item_ct1.get_local_range(2) * item_ct1.get_group_range(2)) {
+        const int   i10 = i0 % ne10;
+        const int   i20 = i0 % ne20;
+        const float acc = bin_op((float) src0_row[i0 * s00], (float) src1_row[i10 * s10]);
+        dst_row[i0]     = (dst_t) bin_op(acc, (float) src2_row[i20 * s20]);
+    }
+}
+
+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename src2_t, typename dst_t>
+static void k_bin_bcast3_unravel(const src0_t * src0, const src1_t * src1, const src2_t * src2, dst_t * dst,
+        int ne0, int ne1, int ne2, int ne3,
+        int ne10, int ne11, int ne12, int ne13,
+        int ne20, int ne21, int ne22, int ne23,
+        int s1, int s2, int s3,
+        int s00, int s01, int s02, int s03,
+        int s10, int s11, int s12, int s13,
+        int s20, int s21, int s22, int s23,
+        const sycl::nd_item<3> & item_ct1) {
+    const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
+                  item_ct1.get_local_id(2);
+
+    const int i3 = i / (ne2 * ne1 * ne0);
+    const int i2 = (i / (ne1 * ne0)) % ne2;
+    const int i1 = (i / ne0) % ne1;
+    const int i0 = i % ne0;
+
+    if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
+        return;
+    }
+
+    const int i11 = i1 % ne11;
+    const int i12 = i2 % ne12;
+    const int i13 = i3 % ne13;
+    const int i21 = i1 % ne21;
+    const int i22 = i2 % ne22;
+    const int i23 = i3 % ne23;
+
+    const size_t i_src0 = i3 * s03 + i2 * s02 + i1 * s01;
+    const size_t i_src1 = i13 * s13 + i12 * s12 + i11 * s11;
+    const size_t i_src2 = i23 * s23 + i22 * s22 + i21 * s21;
+    const size_t i_dst  = i3 * s3 + i2 * s2 + i1 * s1;
+
+    const int   i10 = i0 % ne10;
+    const int   i20 = i0 % ne20;
+    const float acc = bin_op((float) src0[i_src0 + i0 * s00], (float) src1[i_src1 + i10 * s10]);
+    dst[i_dst + i0] = (dst_t) bin_op(acc, (float) src2[i_src2 + i20 * s20]);
+}
+
+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename src2_t, typename dst_t>
+static void launch_bin_bcast3(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1,
+                              const ggml_tensor * src2, ggml_tensor * dst) {
+    dpct::queue_ptr stream = ctx.stream();
+    SYCL_CHECK(ggml_sycl_set_device(ctx.device));
+
+    GGML_TENSOR_TERNARY_OP_LOCALS
+
+    int nr1[4] = { (int) (ne10 / ne0), (int) (ne11 / ne1), (int) (ne12 / ne2), (int) (ne13 / ne3) };
+    int nr2[4] = { (int) (ne20 / ne0), (int) (ne21 / ne1), (int) (ne22 / ne2), (int) (ne23 / ne3) };
+
+    int64_t cne[]  = { ne0, ne1, ne2, ne3 };
+    int64_t cne0[] = { ne00, ne01, ne02, ne03 };
+    int64_t cne1[] = { ne10, ne11, ne12, ne13 };
+    int64_t cne2[] = { ne20, ne21, ne22, ne23 };
+    size_t  cnb[]  = { nb0, nb1, nb2, nb3 };
+    size_t  cnb0[] = { nb00, nb01, nb02, nb03 };
+    size_t  cnb1[] = { nb10, nb11, nb12, nb13 };
+    size_t  cnb2[] = { nb20, nb21, nb22, nb23 };
+
+    auto collapse = [](int64_t cne[]) {
+        cne[0] *= cne[1];
+        cne[1] = cne[2];
+        cne[2] = cne[3];
+        cne[3] = 1;
+    };
+
+    auto collapse_nb = [](size_t cnb[], int64_t cne[]) {
+        cnb[1] *= cne[1];
+        cnb[2] *= cne[2];
+        cnb[3] *= cne[3];
+    };
+
+    const bool can_collapse = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(src2) &&
+                              !ggml_is_permuted(src0) && !ggml_is_permuted(src1) && !ggml_is_permuted(src2);
+    if (can_collapse) {
+        for (int i = 0; i < 4; i++) {
+            if (nr1[i] != 1 || nr2[i] != 1) {
+                break;
+            }
+            if (i > 0) {
+                collapse_nb(cnb, cne);
+                collapse_nb(cnb0, cne0);
+                collapse_nb(cnb1, cne1);
+                collapse_nb(cnb2, cne2);
+                collapse(cne);
+                collapse(cne0);
+                collapse(cne1);
+                collapse(cne2);
+            }
+        }
+    }
+
+    {
+        int64_t ne0 = cne[0];
+        int64_t ne1 = cne[1];
+        int64_t ne2 = cne[2];
+        int64_t ne3 = cne[3];
+
+        int64_t ne10 = cne1[0];
+        int64_t ne11 = cne1[1];
+        int64_t ne12 = cne1[2];
+        int64_t ne13 = cne1[3];
+
+        int64_t ne20 = cne2[0];
+        int64_t ne21 = cne2[1];
+        int64_t ne22 = cne2[2];
+        int64_t ne23 = cne2[3];
+
+        size_t s1 = cnb[1] / sizeof(dst_t);
+        size_t s2 = cnb[2] / sizeof(dst_t);
+        size_t s3 = cnb[3] / sizeof(dst_t);
+
+        size_t s00 = cnb0[0] / sizeof(src0_t);
+        size_t s01 = cnb0[1] / sizeof(src0_t);
+        size_t s02 = cnb0[2] / sizeof(src0_t);
+        size_t s03 = cnb0[3] / sizeof(src0_t);
+
+        size_t s10 = cnb1[0] / sizeof(src1_t);
+        size_t s11 = cnb1[1] / sizeof(src1_t);
+        size_t s12 = cnb1[2] / sizeof(src1_t);
+        size_t s13 = cnb1[3] / sizeof(src1_t);
+
+        size_t s20 = cnb2[0] / sizeof(src2_t);
+        size_t s21 = cnb2[1] / sizeof(src2_t);
+        size_t s22 = cnb2[2] / sizeof(src2_t);
+        size_t s23 = cnb2[3] / sizeof(src2_t);
+
+        GGML_ASSERT(cnb[0] % sizeof(dst_t) == 0 && cnb[1] % sizeof(dst_t) == 0 && cnb[2] % sizeof(dst_t) == 0 &&
+                    cnb[3] % sizeof(dst_t) == 0);
+        GGML_ASSERT(cnb0[0] % sizeof(src0_t) == 0 && cnb0[1] % sizeof(src0_t) == 0 && cnb0[2] % sizeof(src0_t) == 0 &&
+                    cnb0[3] % sizeof(src0_t) == 0);
+        GGML_ASSERT(cnb1[0] % sizeof(src1_t) == 0 && cnb1[1] % sizeof(src1_t) == 0 && cnb1[2] % sizeof(src1_t) == 0 &&
+                    cnb1[3] % sizeof(src1_t) == 0);
+        GGML_ASSERT(cnb2[0] % sizeof(src2_t) == 0 && cnb2[1] % sizeof(src2_t) == 0 && cnb2[2] % sizeof(src2_t) == 0 &&
+                    cnb2[3] % sizeof(src2_t) == 0);
+
+        const src0_t * src0_dd = (const src0_t *) src0->data;
+        const src1_t * src1_dd = (const src1_t *) src1->data;
+        const src2_t * src2_dd = (const src2_t *) src2->data;
+        dst_t *        dst_dd  = (dst_t *) dst->data;
+
+        const int block_size = 128;
+        int64_t   hne0       = std::max(ne0 / 2LL, 1LL);
+
+        sycl::range<3> block_dims(1, 1, 1);
+        block_dims[2] = std::min<unsigned int>(hne0, block_size);
+        block_dims[1] = std::min<unsigned int>(ne1, block_size / (unsigned int) block_dims[2]);
+        block_dims[0] = std::min(std::min<unsigned int>(ne2 * ne3,
+                                                        block_size / (unsigned int) block_dims[2] /
+                                                            (unsigned int) block_dims[1]),
+                                 64U);
+
+        sycl::range<3> block_nums((ne2 * ne3 + block_dims[0] - 1) / block_dims[0],
+                                  (ne1 + block_dims[1] - 1) / block_dims[1],
+                                  (hne0 + block_dims[2] - 1) / block_dims[2]);
+
+        dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
+
+        if (block_nums[0] > 65535) {
+            int block_num = (ne0 * ne1 * ne2 * ne3 + block_size - 1) / block_size;
+            stream->parallel_for(
+                sycl::nd_range<3>(sycl::range<3>(1, 1, block_num) * sycl::range<3>(1, 1, block_size),
+                                  sycl::range<3>(1, 1, block_size)),
+                [=](sycl::nd_item<3> item_ct1) {
+                    k_bin_bcast3_unravel<bin_op>(src0_dd, src1_dd, src2_dd, dst_dd, ne0, ne1, ne2, ne3, ne10, ne11,
+                                                 ne12, ne13, ne20, ne21, ne22, ne23, s1, s2, s3, s00, s01, s02, s03,
+                                                 s10, s11, s12, s13, s20, s21, s22, s23, item_ct1);
+                });
+        } else {
+            stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
+                                 [=](sycl::nd_item<3> item_ct1) {
+                                     k_bin_bcast3<bin_op>(src0_dd, src1_dd, src2_dd, dst_dd, ne0, ne1, ne2, ne3, ne10,
+                                                          ne11, ne12, ne13, ne20, ne21, ne22, ne23, s1, s2, s3, s00,
+                                                          s01, s02, s03, s10, s11, s12, s13, s20, s21, s22, s23,
+                                                          item_ct1);
+                                 });
+        }
+    }
+}
+
+void ggml_sycl_op_add_add_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add0, ggml_tensor * add1) {
+    const ggml_tensor * src0 = add0->src[0];
+    const ggml_tensor * src1 = add0->src[1];
+    const ggml_tensor * src2 = add1->src[1];
+    ggml_tensor *       dst  = add1;
+
+    GGML_ASSERT(add1->src[0] == add0);
+    GGML_ASSERT(ggml_sycl_add_kernel_supports(src0->type, src1->type, add0->type));
+    GGML_ASSERT(ggml_sycl_add_kernel_supports(add0->type, src2->type, dst->type));
+
+    if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_F32 &&
+        dst->type == GGML_TYPE_F32) {
+        launch_bin_bcast3<op_add, float, float, float, float>(ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && src2->type == GGML_TYPE_F16 &&
+               dst->type == GGML_TYPE_F16) {
+        launch_bin_bcast3<op_add, sycl::half, sycl::half, sycl::half, sycl::half>(ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_F32 &&
+               dst->type == GGML_TYPE_F16) {
+        launch_bin_bcast3<op_add, sycl::half, float, float, sycl::half>(ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && src2->type == GGML_TYPE_F32 &&
+               dst->type == GGML_TYPE_F16) {
+        launch_bin_bcast3<op_add, sycl::half, sycl::half, float, sycl::half>(ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_F16 &&
+               dst->type == GGML_TYPE_F16) {
+        launch_bin_bcast3<op_add, sycl::half, float, sycl::half, sycl::half>(ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32 && src2->type == GGML_TYPE_I32 &&
+               dst->type == GGML_TYPE_I32) {
+        launch_bin_bcast3<op_add, int32_t, int32_t, int32_t, int32_t>(ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16 && src2->type == GGML_TYPE_I16 &&
+               dst->type == GGML_TYPE_I16) {
+        launch_bin_bcast3<op_add, int16_t, int16_t, int16_t, int16_t>(ctx, src0, src1, src2, dst);
+#ifdef GGML_SYCL_HAS_BF16
+    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && src2->type == GGML_TYPE_BF16 &&
+               dst->type == GGML_TYPE_BF16) {
+        launch_bin_bcast3<op_add, sycl::ext::oneapi::bfloat16, sycl::ext::oneapi::bfloat16,
+                          sycl::ext::oneapi::bfloat16, sycl::ext::oneapi::bfloat16>(ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_F32 &&
+               dst->type == GGML_TYPE_BF16) {
+        launch_bin_bcast3<op_add, sycl::ext::oneapi::bfloat16, float, float, sycl::ext::oneapi::bfloat16>(
+            ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && src2->type == GGML_TYPE_F32 &&
+               dst->type == GGML_TYPE_BF16) {
+        launch_bin_bcast3<op_add, sycl::ext::oneapi::bfloat16, sycl::ext::oneapi::bfloat16, float,
+                          sycl::ext::oneapi::bfloat16>(ctx, src0, src1, src2, dst);
+    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_BF16 &&
+               dst->type == GGML_TYPE_BF16) {
+        launch_bin_bcast3<op_add, sycl::ext::oneapi::bfloat16, float, sycl::ext::oneapi::bfloat16,
+                          sycl::ext::oneapi::bfloat16>(ctx, src0, src1, src2, dst);
+#endif
+    } else {
+        fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s, src2: %s\n", __func__,
+                ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type),
+                ggml_type_name(src2->type));
+        GGML_ABORT("fatal error");
+    }
+}
+
@@ -34,6 +34,36 @@ void ggml_sycl_div(ggml_backend_sycl_context & ctx, ggml_tensor * dst);

 void ggml_sycl_repeat(ggml_backend_sycl_context & ctx, ggml_tensor * dst);

+void ggml_sycl_op_add_add_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add0, ggml_tensor * add1);
+
+// Type combinations the standalone SYCL add() kernel can run. Fused ADD+ADD
+// uses the same set; anything else falls back to two add() launches.
+inline bool ggml_sycl_add_kernel_supports(enum ggml_type src0, enum ggml_type src1, enum ggml_type dst) {
+    if (src0 == GGML_TYPE_F32 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_F32) {
+        return true;
+    }
+    if (src0 == GGML_TYPE_F16 && src1 == GGML_TYPE_F16 && dst == GGML_TYPE_F16) {
+        return true;
+    }
+    if (src0 == GGML_TYPE_F16 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_F16) {
+        return true;
+    }
+    if (src0 == GGML_TYPE_I32 && src1 == GGML_TYPE_I32 && dst == GGML_TYPE_I32) {
+        return true;
+    }
+    if (src0 == GGML_TYPE_I16 && src1 == GGML_TYPE_I16 && dst == GGML_TYPE_I16) {
+        return true;
+    }
+#ifdef GGML_SYCL_HAS_BF16
+    if (src0 == GGML_TYPE_BF16 && src1 == GGML_TYPE_BF16 && dst == GGML_TYPE_BF16) {
+        return true;
+    }
+    if (src0 == GGML_TYPE_BF16 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_BF16) {
+        return true;
+    }
+#endif
+    return false;
+}

 #endif //GGML_SYCL_BINBCAST_HPP

@@ -94,7 +94,7 @@ static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) {

 // Use Level Zero zeMemAllocDevice to avoid sycl::malloc_device triggering
 // DMA-buf/TTM system RAM staging in the xe kernel driver during multi-GPU inference.
-void * ggml_sycl_malloc_device(size_t size, sycl::queue &q) {
+void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) {
 #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
    if (ggml_sycl_use_level_zero_device_alloc(q)) {
        void *ptr = nullptr;
@@ -117,16 +117,25 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q) {
 #endif
        ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, 64, ze_dev, &ptr);
        if (r == ZE_RESULT_SUCCESS && ptr) {
+            ggml_sycl_memtrace_add(type, ptr, size);
            return ptr;
        }
+        ggml_sycl_memtrace_fail(type, size);
        return nullptr;
    }
 #endif
-    return sycl::malloc_device(size, q);
+    void * ptr = sycl::malloc_device(size, q);
+    if (ptr == nullptr) {
+        ggml_sycl_memtrace_fail(type, size);
+        return nullptr;
+    }
+    ggml_sycl_memtrace_add(type, ptr, size);
+    return ptr;
 }

 void ggml_sycl_free_device(void *ptr, sycl::queue &q) {
    if (!ptr) return;
+    ggml_sycl_memtrace_del(ptr);
 #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
    if (ggml_sycl_use_level_zero_device_alloc(q)) {
        auto ze_ctx = sycl::get_native<sycl::backend::ext_oneapi_level_zero>(q.get_context());
@@ -27,6 +27,7 @@
 #include "type.hpp"
 #include "sycl_hw.hpp"
 #include "fattn-buffers.hpp"
+#include "memtrace.hpp"

 namespace syclexp = sycl::ext::oneapi::experimental;

@@ -68,6 +69,9 @@ extern int g_ggml_sycl_enable_flash_attention;
 extern int g_ggml_sycl_dev2dev_memcpy;
 extern int g_ggml_sycl_fa_onednn;
 extern int g_ggml_sycl_fa_onednn_max_kv;
+extern int g_ggml_sycl_enable_mkl_fa;
+extern int g_ggml_sycl_memtrace;
+extern int g_ggml_sycl_memtrace_step;


 #define CHECK_TRY_ERROR(expr)                                            \
@@ -317,7 +321,8 @@ struct ggml_tensor_extra_gpu {
 };

 extern int g_ggml_sycl_use_level_zero_api;
-void * ggml_sycl_malloc_device(size_t size, sycl::queue &q);
+void * ggml_sycl_malloc_device(size_t size, sycl::queue &q,
+                               ggml_sycl_mem_type type = GGML_SYCL_MEM_DIRECT);
 void ggml_sycl_free_device(void *ptr, sycl::queue &q);

 void release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector<queue_ptr> streams={});
--- a/Show More
+++ b/Show More