naming : improve consistency

tools : update readme
server : fix slot ctx_drft ptr
2026-07-17 01:45:59 +02:00 · 2026-05-08 12:24:57 +03:00 · 2026-05-08 11:55:16 +03:00 · 2026-05-08 11:55:05 +03:00 · 2026-05-08 11:54:32 +03:00 · 2026-05-08 11:09:34 +03:00
306 changed files with 28834 additions and 41075 deletions
@@ -5,15 +5,8 @@ ARG ONEAPI_VERSION=2025.3.3-0-devel-ubuntu24.04
 FROM intel/deep-learning-essentials:$ONEAPI_VERSION AS build

 ARG GGML_SYCL_F16=OFF
-ARG LEVEL_ZERO_VERSION=1.28.2
-ARG LEVEL_ZERO_UBUNTU_VERSION=u24.04
 RUN apt-get update && \
-    apt-get install -y git libssl-dev wget ca-certificates && \
-    cd /tmp && \
-    wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb && \
-    wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb && \
-    apt-get -o Dpkg::Options::="--force-overwrite" install -y ./level-zero.deb ./level-zero-devel.deb && \
-    rm -f /tmp/level-zero.deb /tmp/level-zero-devel.deb
+    apt-get install -y git libssl-dev

 WORKDIR /app

@@ -40,11 +33,11 @@ RUN mkdir -p /app/full \

 FROM intel/deep-learning-essentials:$ONEAPI_VERSION AS base

-ARG IGC_VERSION=v2.20.5
-ARG IGC_VERSION_FULL=2_2.20.5+19972
-ARG COMPUTE_RUNTIME_VERSION=25.40.35563.10
-ARG COMPUTE_RUNTIME_VERSION_FULL=25.40.35563.10-0
-ARG IGDGMM_VERSION=22.8.2
+ARG IGC_VERSION=v2.30.1
+ARG IGC_VERSION_FULL=2_2.30.1+20950
+ARG COMPUTE_RUNTIME_VERSION=26.09.37435.1
+ARG COMPUTE_RUNTIME_VERSION_FULL=26.09.37435.1-0
+ARG IGDGMM_VERSION=22.9.0
 RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
  && wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \
  && wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \
@@ -116,3 +109,4 @@ WORKDIR /app
 HEALTHCHECK CMD [ "curl", "-f", "http://localhost:8080/health" ]

 ENTRYPOINT [ "/app/llama-server" ]
+
@@ -103,7 +103,6 @@ let
    vulkan-headers
    vulkan-loader
    shaderc
-    spirv-headers
  ];
 in

@@ -147,6 +146,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
      ninja
      pkg-config
      git
+      spirv-headers
    ]
    ++ optionals useCuda [
      cudaPackages.cuda_nvcc
@@ -53,6 +53,14 @@ charset = unset
 trim_trailing_whitespace = unset
 insert_final_newline = unset

+[tools/server/public/**]
+indent_style = unset
+indent_size = unset
+end_of_line = unset
+charset = unset
+trim_trailing_whitespace = unset
+insert_final_newline = unset
+
 [benches/**]
 indent_style = unset
 indent_size = unset
@@ -0,0 +1,4 @@
+# Treat the generated single-file WebUI build as binary for diff purposes.
+# Git's pack-file delta compression still works (byte-level), but this prevents
+# git diff from printing the entire minified file on every change.
+tools/server/public/index.html -diff
@@ -77,6 +77,7 @@ server/webui:
    - changed-files:
        - any-glob-to-any-file:
            - tools/server/webui/**
+            - tools/server/public/**
 server:
    - changed-files:
        - any-glob-to-any-file:
@@ -58,45 +58,14 @@ jobs:
          name: llama-cpp-android-arm64-snapdragon
          path: pkg-snapdragon/llama.cpp

-  linux-iot-snapdragon:
-    runs-on: ubuntu-latest
-    container:
-      image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.1'
-    defaults:
-      run:
-        shell: bash
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-          lfs: false
-
-      - name: Build Llama.CPP for Snapdragon Linux IoT
-        id: build_llama_cpp_snapdragon_linux
-        run: |
-          cp docs/backend/snapdragon/CMakeUserPresets.json .
-          cmake --preset arm64-linux-snapdragon-release -B build-snapdragon -DGGML_OPENCL=ON
-          cmake --build build-snapdragon -j $(nproc)
-          cmake --install build-snapdragon --prefix pkg-snapdragon/llama.cpp
-
-      - name: Upload Llama.CPP Snapdragon Linux IoT Build Artifact
-        if: ${{ always() && steps.build_llama_cpp_snapdragon_linux.outcome == 'success' }}
-        uses: actions/upload-artifact@v6
-        with:
-          name: llama-cpp-linux-arm64-snapdragon
-          path: pkg-snapdragon/llama.cpp
-
  test-snapdragon-qdc:
-    name: Test on QDC Device (${{ matrix.device }})
-    needs: [android-ndk-snapdragon, linux-iot-snapdragon]
-    runs-on: ubuntu-24.04-arm
-    timeout-minutes: 90
+    name: Test on QDC Android Device (${{ matrix.device }})
+    needs: [android-ndk-snapdragon]
+    runs-on: ubuntu-slim
    strategy:
      fail-fast: false
      matrix:
-        device: [SM8750, SM8850, QCS9075M]
+        device: [SM8750, SM8650, SM8850]

    steps:
      - name: Checkout
@@ -105,11 +74,11 @@ jobs:
      - name: Download build artifact
        uses: actions/download-artifact@v7
        with:
-          name: ${{ startsWith(matrix.device, 'QCS') && 'llama-cpp-linux-arm64-snapdragon' || 'llama-cpp-android-arm64-snapdragon' }}
+          name: llama-cpp-android-arm64-snapdragon
          path: pkg-snapdragon/llama.cpp

      - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
        with:
          python-version: '3.x'
          cache: pip
@@ -138,8 +107,7 @@ jobs:
              --test       all \
              --pkg-dir    pkg-snapdragon/llama.cpp \
              --model-url  "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" \
-              --device     ${{ matrix.device }} \
-              ${{ startsWith(matrix.device, 'QCS') && '--retries 2 --retry-delay 300' || '' }}
+              --device     ${{ matrix.device }}
        env:
          QDC_API_KEY: ${{ secrets.QDC_API_KEY }}

@@ -301,17 +301,16 @@ jobs:
          export RISCV_ROOT_PATH=${PWD}/spacemit_toolchain
          cmake -B build -DLLAMA_OPENSSL=OFF \
                         -DCMAKE_BUILD_TYPE=Release \
+                         -DGGML_OPENMP=OFF \
                         -DLLAMA_BUILD_EXAMPLES=ON \
-                         -DGGML_CPU_REPACK=OFF \
                         -DLLAMA_BUILD_TOOLS=ON \
                         -DLLAMA_BUILD_TESTS=OFF \
                         -DGGML_CPU_RISCV64_SPACEMIT=ON \
                         -DGGML_RVV=ON \
-                         -DGGML_RV_ZVFH=ON \
                         -DGGML_RV_ZFH=ON \
                         -DGGML_RV_ZICBOP=ON \
                         -DGGML_RV_ZIHINTPAUSE=ON \
-                         -DGGML_RV_ZBA=ON \
+                         -DRISCV64_SPACEMIT_IME_SPEC=RISCV64_SPACEMIT_IME1 \
                         -DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake

          cmake --build build --config Release -j $(nproc)
@@ -55,22 +55,7 @@ env:
  LLAMA_LOG_TIMESTAMPS: 1

 jobs:
-  determine-tag:
-    name: Determine tag name
-    runs-on: ubuntu-slim
-    outputs:
-      tag_name: ${{ steps.tag.outputs.name }}
-    steps:
-      - name: Clone
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-      - name: Determine tag name
-        id: tag
-        uses: ./.github/actions/get-tag-name
-
  ggml-ci-nvidia-cuda:
-    needs: determine-tag
    runs-on: [self-hosted, Linux, NVIDIA]

    steps:
@@ -80,14 +65,11 @@ jobs:

      - name: Test
        id: ggml-ci
-        env:
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          nvidia-smi
          GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp

  ggml-ci-nvidia-vulkan-cm:
-    needs: determine-tag
    runs-on: [self-hosted, Linux, NVIDIA]

    steps:
@@ -97,14 +79,11 @@ jobs:

      - name: Test
        id: ggml-ci
-        env:
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          vulkaninfo --summary
          GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp

  ggml-ci-nvidia-vulkan-cm2:
-    needs: determine-tag
    runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]

    steps:
@@ -114,40 +93,39 @@ jobs:

      - name: Test
        id: ggml-ci
-        env:
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          vulkaninfo --summary
          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp

-  ggml-ci-nvidia-webgpu:
-    runs-on: [self-hosted, Linux, NVIDIA]
+  # TODO: investigate slight precision issues in some operations for test-backend-ops on the WebGPU backend.
+  #ggml-ci-nvidia-webgpu:
+  #  runs-on: [self-hosted, Linux, NVIDIA]

-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
+  #  steps:
+  #    - name: Clone
+  #      id: checkout
+  #      uses: actions/checkout@v6

-      - name: Dawn Dependency
-        id: dawn-depends
-        run: |
-          DAWN_VERSION="v20260317.182325"
-          DAWN_OWNER="google"
-          DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
-          echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
-          curl -L -o artifact.tar.gz \
-            "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
-          mkdir dawn
-          tar -xvf artifact.tar.gz -C dawn --strip-components=1
+  #    - name: Dawn Dependency
+  #      id: dawn-depends
+  #      run: |
+  #        DAWN_VERSION="v20260317.182325"
+  #        DAWN_OWNER="google"
+  #        DAWN_REPO="dawn"
+  #        DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
+  #        echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+  #        curl -L -o artifact.tar.gz \
+  #          "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+  #        mkdir dawn
+  #        tar -xvf artifact.tar.gz -C dawn --strip-components=1

-      - name: Test
-        id: ggml-ci
-        run: |
-          GG_BUILD_WEBGPU=1 \
-          GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
-          GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
-            bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
+  #    - name: Test
+  #      id: ggml-ci
+  #      run: |
+  #        GG_BUILD_WEBGPU=1 \
+  #        GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
+  #        GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
+  #          bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp

  # TODO: provision AMX-compatible machine
  #ggml-ci-cpu-amx:
@@ -194,7 +172,6 @@ jobs:
  #         GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp

  ggml-ci-mac-metal:
-    needs: determine-tag
    runs-on: [self-hosted, macOS, ARM64]

    steps:
@@ -204,13 +181,10 @@ jobs:

      - name: Test
        id: ggml-ci
-        env:
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

  ggml-ci-mac-webgpu:
-    needs: determine-tag
    runs-on: [self-hosted, macOS, ARM64]

    steps:
@@ -233,14 +207,11 @@ jobs:

      - name: Test
        id: ggml-ci
-        env:
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
            bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

  ggml-ci-mac-vulkan:
-    needs: determine-tag
    runs-on: [self-hosted, macOS, ARM64]

    steps:
@@ -250,14 +221,11 @@ jobs:

      - name: Test
        id: ggml-ci
-        env:
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          vulkaninfo --summary
          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

  ggml-ci-linux-intel-vulkan:
-    needs: determine-tag
    runs-on: [self-hosted, Linux, Intel]

    steps:
@@ -269,14 +237,11 @@ jobs:

      - name: Test
        id: ggml-ci
-        env:
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          vulkaninfo --summary
          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

  ggml-ci-win-intel-vulkan:
-    needs: determine-tag
    runs-on: [self-hosted, Windows, X64, Intel]

    steps:
@@ -291,7 +256,6 @@ jobs:
          MSYSTEM: UCRT64
          CHERE_INVOKING: 1
          PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          vulkaninfo --summary
          # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
@@ -299,7 +263,6 @@ jobs:
          LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp

  ggml-ci-intel-openvino-gpu-low-perf:
-    needs: determine-tag
    runs-on: [self-hosted, Linux, Intel, OpenVINO]

    concurrency:
@@ -331,8 +294,6 @@ jobs:

      - name: Test
        id: ggml-ci
-        env:
-          HF_WEBUI_VERSION: ${{ needs.determine-tag.outputs.tag_name }}
        run: |
          source ./openvino_toolkit/setupvars.sh
          GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
@@ -50,8 +50,6 @@ jobs:
    env:
      ONEAPI_ROOT: /opt/intel/oneapi/
      ONEAPI_INSTALLER_VERSION: "2025.3.3"
-      LEVEL_ZERO_VERSION: "1.28.2"
-      LEVEL_ZERO_UBUNTU_VERSION: "u24.04"

    continue-on-error: true

@@ -73,14 +71,6 @@ jobs:
          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
          sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept

-      - name: Install Level Zero SDK
-        shell: bash
-        run: |
-          cd /tmp
-          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
-          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
-          sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
-
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
@@ -117,7 +107,6 @@ jobs:
    env:
      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
-      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
      ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
      ONEAPI_INSTALLER_VERSION: "2025.3.3"
    steps:
@@ -138,13 +127,6 @@ jobs:
        run: |
          scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL

-      - name: Install Level Zero SDK
-        shell: pwsh
-        run: |
-          Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile "level-zero-win-sdk.zip"
-          Expand-Archive -Path "level-zero-win-sdk.zip" -DestinationPath "C:/level-zero-sdk" -Force
-          "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -1,50 +0,0 @@
-name: CI (virtgpu)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/build-virtgpu.yml',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/build-virtgpu.yml',
-      'ggml/src/ggml-virtgpu/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  ubuntu-24-virtgpu:
-    runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Dependencies
-        id: depends
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential libdrm-dev pkg-config libssl-dev
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DGGML_VIRTGPU=ON \
-            -DGGML_VIRTGPU_BACKEND=ON
-          cmake --build build --config Release -j $(nproc)
@@ -456,8 +456,7 @@ jobs:
        run: |
          cd build
          # This is using llvmpipe and runs slower than other backends
-          # test-backend-ops is too slow on llvmpipe, skip it
-          ctest -L main -E test-backend-ops --verbose --timeout 900
+          ctest -L main --verbose --timeout 900

  ubuntu-24-webgpu-wasm:
    runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
@@ -1,51 +0,0 @@
-name: Code Style Checker
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  model-naming:
-    runs-on: ubuntu-slim
-    steps:
-      - uses: actions/checkout@v6
-      - name: Check model naming conventions
-        run: |
-          python3 - << 'EOF'
-          import re, os, sys
-
-          pairs = re.findall(
-              r'case\s+(LLM_ARCH_\w+)\s*:\s*\n\s+return new (llama_model_\w+)\s*\(',
-              open("src/llama-model.cpp").read())
-
-          errors = []
-          for arch, cls in pairs:
-              suffix  = arch[len("LLM_ARCH_"):]
-              csuffix = cls[len("llama_model_"):]
-              fname   = csuffix.replace("_", "-") + ".cpp"
-
-              if not re.fullmatch(r'[A-Z][A-Z0-9_]*',   suffix):
-                  errors.append(f"{arch}: suffix not upper snake case, example: LLM_ARCH_MY_MODEL")
-
-              if not re.fullmatch(r'[a-z][a-z0-9_]*', csuffix):
-                  errors.append(f"{arch}: class suffix not lower snake case, example: llama_model_my_model")
-
-              elif suffix.lower() != csuffix:
-                  errors.append(f"{arch}: arch/class name mismatch, expected class 'llama_model_{suffix.lower()}' but got '{cls}'")
-
-              elif not os.path.isfile(f"src/models/{fname}"):
-                  errors.append(f"{arch}: expects model file name to be src/models/{fname}, but not found")
-
-          if errors:
-              print('\n'.join(f"  - {e}" for e in errors)); sys.exit(1)
-          print(f"OK: {len(pairs)} mappings validated.")
-          EOF
@@ -2,6 +2,11 @@ name: EditorConfig Checker

 on:
  workflow_dispatch: # allows manual triggering
+    inputs:
+      create_release:
+        description: 'Create new release'
+        required: true
+        type: boolean
  push:
    branches:
      - master
@@ -31,7 +31,7 @@ jobs:
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"
-          pip-install: -r requirements/requirements-all.txt ty==0.0.35
+          pip-install: -r requirements/requirements-all.txt ty==0.0.33
      # - name: Type-check with Pyright
      #   uses: jakebailey/pyright-action@v2
      #   with:
@@ -36,14 +36,7 @@ env:
  CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"

 jobs:
-  webui-build:
-    name: Build WebUI
-    uses: ./.github/workflows/webui-build.yml
-
  macOS-cpu:
-    needs:
-      - webui-build
-
    strategy:
      matrix:
        include:
@@ -71,12 +64,6 @@ jobs:
        with:
          fetch-depth: 0

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -104,7 +91,7 @@ jobs:
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz -s ",^.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz -s ",./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
@@ -113,9 +100,6 @@ jobs:
          name: llama-bin-macos-${{ matrix.build }}.tar.gz

  ubuntu-cpu:
-    needs:
-      - webui-build
-
    strategy:
      matrix:
        include:
@@ -135,12 +119,6 @@ jobs:
        with:
          fetch-depth: 0

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        if: ${{ matrix.build != 's390x' }}
        uses: ggml-org/ccache-action@v1.2.21
@@ -182,7 +160,7 @@ jobs:
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz --transform "s,^.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
@@ -191,9 +169,6 @@ jobs:
          name: llama-bin-ubuntu-${{ matrix.build }}.tar.gz

  ubuntu-vulkan:
-    needs:
-      - webui-build
-
    strategy:
      matrix:
        include:
@@ -211,12 +186,6 @@ jobs:
        with:
          fetch-depth: 0

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -259,7 +228,7 @@ jobs:
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz --transform "s,^.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
@@ -268,9 +237,6 @@ jobs:
          name: llama-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz

  android-arm64:
-    needs:
-      - webui-build
-
    runs-on: ubuntu-latest

    env:
@@ -283,12 +249,6 @@ jobs:
        with:
          fetch-depth: 0

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -337,7 +297,7 @@ jobs:
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz --transform "s,^.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
@@ -346,9 +306,6 @@ jobs:
          name: llama-bin-android-arm64.tar.gz

  ubuntu-24-openvino:
-    needs:
-      - webui-build
-
    runs-on: ubuntu-24.04

    outputs:
@@ -370,12 +327,6 @@ jobs:
        with:
          fetch-depth: 0

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -426,7 +377,7 @@ jobs:
        id: pack_artifacts
        run: |
          cp LICENSE ./build/ReleaseOV/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,^.,llama-${{ steps.tag.outputs.name }}," -C ./build/ReleaseOV/bin .
+          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/ReleaseOV/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
@@ -435,9 +386,6 @@ jobs:
          name: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz

  windows-cpu:
-    needs:
-      - webui-build
-
    runs-on: windows-2025

    strategy:
@@ -452,12 +400,6 @@ jobs:
        with:
          fetch-depth: 0

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -496,9 +438,6 @@ jobs:
          name: llama-bin-win-cpu-${{ matrix.arch }}.zip

  windows:
-    needs:
-      - webui-build
-
    runs-on: windows-2025

    env:
@@ -522,12 +461,6 @@ jobs:
        id: checkout
        uses: actions/checkout@v6

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -587,9 +520,6 @@ jobs:
          name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip

  windows-cuda:
-    needs:
-      - webui-build
-
    runs-on: windows-2022

    strategy:
@@ -601,12 +531,6 @@ jobs:
        id: checkout
        uses: actions/checkout@v6

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: Install ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -667,9 +591,6 @@ jobs:
          name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip

  windows-sycl:
-    needs:
-      - webui-build
-
    runs-on: windows-2022

    defaults:
@@ -679,7 +600,6 @@ jobs:
    env:
      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
-      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
      ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
      ONEAPI_INSTALLER_VERSION: "2025.3.3"

@@ -701,19 +621,6 @@ jobs:
        run: |
          scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL

-      - name: Install Level Zero SDK
-        shell: pwsh
-        run: |
-          Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile "level-zero-win-sdk.zip"
-          Expand-Archive -Path "level-zero-win-sdk.zip" -DestinationPath "C:/level-zero-sdk" -Force
-          "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
-
-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -748,13 +655,6 @@ jobs:
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_opencl.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_win_proxy_loader.dll" ./build/bin
-          ZE_LOADER_DLL=$(find "${{ env.ONEAPI_ROOT }}" "$LEVEL_ZERO_V1_SDK_PATH" -iname ze_loader.dll -print -quit 2>/dev/null || true)
-          if [ -n "$ZE_LOADER_DLL" ]; then
-            echo "Using Level Zero loader: $ZE_LOADER_DLL"
-            cp "$ZE_LOADER_DLL" ./build/bin
-          else
-            echo "Level Zero loader DLL not found in oneAPI or SDK; relying on system driver/runtime"
-          fi

          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl8.dll" ./build/bin
          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/svml_dispmd.dll" ./build/bin
@@ -781,9 +681,6 @@ jobs:
          name: llama-bin-win-sycl-x64.zip

  ubuntu-24-sycl:
-    needs:
-      - webui-build
-
    strategy:
      matrix:
        build: [fp32, fp16]
@@ -798,8 +695,6 @@ jobs:
    env:
      ONEAPI_ROOT: /opt/intel/oneapi/
      ONEAPI_INSTALLER_VERSION: "2025.3.3"
-      LEVEL_ZERO_VERSION: "1.28.2"
-      LEVEL_ZERO_UBUNTU_VERSION: "u24.04"

    steps:
      - name: Clone
@@ -823,20 +718,6 @@ jobs:
          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
          sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept

-      - name: Install Level Zero SDK
-        shell: bash
-        run: |
-          cd /tmp
-          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
-          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
-          sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
-
-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
@@ -867,7 +748,7 @@ jobs:
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz --transform "s,^.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
@@ -876,9 +757,6 @@ jobs:
          name: llama-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz

  ubuntu-22-rocm:
-    needs:
-      - webui-build
-
    runs-on: ubuntu-22.04

    strategy:
@@ -895,12 +773,6 @@ jobs:
        with:
          fetch-depth: 0

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: Free up disk space
        uses: ggml-org/free-disk-space@v1.3.1
        with:
@@ -979,7 +851,7 @@ jobs:
        id: pack_artifacts
        run: |
          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
@@ -988,9 +860,6 @@ jobs:
          name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz

  windows-hip:
-    needs:
-      - webui-build
-
    runs-on: windows-2022

    env:
@@ -1007,12 +876,6 @@ jobs:
        id: checkout
        uses: actions/checkout@v6

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: Grab rocWMMA package
        id: grab_rocwmma
        run: |
@@ -1240,7 +1103,7 @@ jobs:
      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz --transform "s,^.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz --transform "s,./,llama-${{ steps.tag.outputs.name }}/," -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
@@ -1259,7 +1122,6 @@ jobs:
    runs-on: ubuntu-slim

    needs:
-      - webui-build
      - windows
      - windows-cpu
      - windows-cuda
@@ -1275,9 +1137,6 @@ jobs:
      - ios-xcode-build
      - openEuler-cann

-    outputs:
-      tag_name: ${{ steps.tag.outputs.name }}
-
    steps:
      - name: Clone
        id: checkout
@@ -1403,15 +1262,3 @@ jobs:
                });
              }
            }
-
-  webui-publish:
-    if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-
-    needs:
-      - release
-
-    uses: ./.github/workflows/webui-publish.yml
-    with:
-      version_tag: ${{ needs.release.outputs.tag_name }}
-    secrets:
-      hf_token: ${{ secrets.HF_TOKEN_WEBUI_STATIC_OUTPUT }}
@@ -39,12 +39,7 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  webui-build:
-    name: Build WebUI
-    uses: ./.github/workflows/webui-build.yml
-
  server-metal:
-    needs: webui-build
    runs-on: [self-hosted, llama-server, macOS, ARM64]

    name: server-metal (${{ matrix.wf_name }})
@@ -72,12 +67,6 @@ jobs:
          fetch-depth: 0
          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: Build
        id: cmake_build
        run: |
@@ -1,7 +1,7 @@
 name: Server WebUI

 on:
-  workflow_dispatch:
+  workflow_dispatch: # allows manual triggering
    inputs:
      sha:
        description: 'Commit SHA1 to build'
@@ -13,14 +13,16 @@ on:
    paths: [
      '.github/workflows/server-webui.yml',
      'tools/server/webui/**.*',
-      'tools/server/tests/**.*'
+      'tools/server/tests/**.*',
+      'tools/server/public/**'
    ]
  pull_request:
    types: [opened, synchronize, reopened]
    paths: [
      '.github/workflows/server-webui.yml',
      'tools/server/webui/**.*',
-      'tools/server/tests/**.*'
+      'tools/server/tests/**.*',
+      'tools/server/public/**'
    ]

 env:
@@ -34,14 +36,9 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  webui-build:
-    name: Build WebUI
-    uses: ./.github/workflows/webui-build.yml
-
-  webui-checks:
+  webui-check:
    name: WebUI Checks
-    needs: webui-build
-    runs-on: ubuntu-24.04-arm
+    runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
    continue-on-error: true
    steps:
      - name: Checkout code
@@ -54,7 +51,7 @@ jobs:
        id: node
        uses: actions/setup-node@v6
        with:
-          node-version: "24"
+          node-version: "22"
          cache: "npm"
          cache-dependency-path: "tools/server/webui/package-lock.json"

@@ -74,47 +71,6 @@ jobs:
        run: npm run lint
        working-directory: tools/server/webui

-      - name: Install Playwright browsers
-        id: playwright
-        if: ${{ always() && steps.setup.conclusion == 'success' }}
-        run: npx playwright install --with-deps
-        working-directory: tools/server/webui
-
-      - name: Run Client tests
-        if: ${{ always() && steps.playwright.conclusion == 'success' }}
-        run: npm run test:client
-        working-directory: tools/server/webui
-
-      - name: Run Unit tests
-        if: ${{ always() && steps.playwright.conclusion == 'success' }}
-        run: npm run test:unit
-        working-directory: tools/server/webui
-
-  e2e-tests:
-    name: E2E Tests
-    needs: webui-build
-    runs-on: ubuntu-24.04-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
-
-      - name: Setup Node.js
-        id: node
-        uses: actions/setup-node@v6
-        with:
-          node-version: "24"
-          cache: "npm"
-          cache-dependency-path: "tools/server/webui/package-lock.json"
-
-      - name: Install dependencies
-        id: setup
-        if: ${{ steps.node.conclusion == 'success' }}
-        run: npm ci
-        working-directory: tools/server/webui
-
      - name: Build application
        if: ${{ always() && steps.setup.conclusion == 'success' }}
        run: npm run build
@@ -131,6 +87,16 @@ jobs:
        run: npm run build-storybook
        working-directory: tools/server/webui

+      - name: Run Client tests
+        if: ${{ always() && steps.playwright.conclusion == 'success' }}
+        run: npm run test:client
+        working-directory: tools/server/webui
+
+      - name: Run Unit tests
+        if: ${{ always() && steps.playwright.conclusion == 'success' }}
+        run: npm run test:unit
+        working-directory: tools/server/webui
+
      - name: Run UI tests
        if: ${{ always() && steps.playwright.conclusion == 'success' }}
        run: npm run test:ui -- --testTimeout=60000
@@ -54,12 +54,7 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  webui-build:
-    name: Build WebUI
-    uses: ./.github/workflows/webui-build.yml
-
  server:
-    needs: webui-build
    runs-on: ubuntu-latest

    name: server (${{ matrix.wf_name }})
@@ -98,12 +93,6 @@ jobs:
          fetch-depth: 0
          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: Build
        id: cmake_build
        run: |
@@ -136,7 +125,6 @@ jobs:
          SLOW_TESTS=1 pytest -v -x

  server-windows:
-    needs: webui-build
    runs-on: windows-2022

    steps:
@@ -147,12 +135,6 @@ jobs:
          fetch-depth: 0
          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}

-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
      - name: Build
        id: cmake_build
        run: |
@@ -1,44 +0,0 @@
-name: Build WebUI
-
-on:
-  workflow_call:
-
-jobs:
-  build:
-    name: Build WebUI
-    runs-on: ubuntu-slim
-    env:
-      BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: "24"
-          cache: "npm"
-          cache-dependency-path: "tools/server/webui/package-lock.json"
-
-      - name: Install dependencies
-        run: npm ci
-        working-directory: tools/server/webui
-
-      - name: Build application
-        run: npm run build
-        working-directory: tools/server/webui
-
-      - name: Generate checksums
-        run: |
-          cd tools/server/public
-          for f in *; do
-            sha256sum "$f" | awk '{print $1, $2}' >> checksums.txt
-          done
-
-      - name: Upload built webui
-        uses: actions/upload-artifact@v6
-        with:
-          name: webui-build
-          path: tools/server/public/
-          retention-days: 1
@@ -1,65 +0,0 @@
-name: WebUI Publish
-
-on:
-  workflow_call:
-    inputs:
-      version_tag:
-        description: 'Version tag to publish under (e.g., b1234)'
-        required: true
-        type: string
-    secrets:
-      hf_token:
-        description: 'Hugging Face token with write access'
-        required: true
-
-jobs:
-  publish:
-    name: Publish WebUI Static Output
-    runs-on: ubuntu-24.04-arm
-
-    permissions:
-      contents: read
-
-    env:
-      HF_BUCKET_NAME: ${{ vars.HF_BUCKET_WEBUI_STATIC_OUTPUT }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 1
-
-      - name: Download WebUI build artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: webui-build
-          path: tools/server/public/
-
-      - name: Install Hugging Face Hub CLI
-        run: pip install -U huggingface_hub
-
-      - name: Authenticate with Hugging Face
-        run: hf auth login --token ${{ secrets.hf_token }}
-
-      - name: Sync built files to Hugging Face bucket (version tag)
-        run: |
-          # Upload the built files to the Hugging Face bucket under the release version
-          hf buckets sync tools/server/public hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
-
-      - name: Sync built files to Hugging Face bucket (latest)
-        run: |
-          # Also upload to the 'latest' directory for fallback downloads
-          hf buckets sync tools/server/public hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
-
-      - name: Verify upload
-        run: |
-          # List the files in the bucket to verify the upload
-          hf buckets list hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} -R -h
-
-      - name: Clean up root-level files
-        run: |
-          # Clean up any old root-level files from previous non-versioned deployments
-          hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/index.html --yes 2>/dev/null || true
-          hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/bundle.js --yes 2>/dev/null || true
-          hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/bundle.css --yes 2>/dev/null || true
-          hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/loading.html --yes 2>/dev/null || true
@@ -54,7 +54,6 @@
 /tmp/
 /autogen-*.md
 /common/build-info.cpp
-/tools/server/public

 # Deprecated

@@ -97,6 +96,8 @@

 /tools/server/webui/node_modules
 /tools/server/webui/dist
+# we no longer use gz for index.html
+/tools/server/public/index.html.gz

 # Python

@@ -109,7 +110,6 @@ uv.lock

 # Nix

-flake.lock
 /result

 # Test binaries
@@ -104,14 +104,13 @@ option(LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" OFF)
 option(LLAMA_BUILD_COMMON "llama: build common utils library" ${LLAMA_STANDALONE})

 # extra artifacts
-option(LLAMA_BUILD_TESTS            "llama: build tests"                                                                            ${LLAMA_STANDALONE})
-option(LLAMA_BUILD_TOOLS            "llama: build tools"                                                                            ${LLAMA_STANDALONE})
-option(LLAMA_BUILD_EXAMPLES         "llama: build examples"                                                                         ${LLAMA_STANDALONE})
-option(LLAMA_BUILD_SERVER           "llama: build server example"                                                                   ${LLAMA_STANDALONE})
-option(LLAMA_BUILD_WEBUI            "llama: build the embedded Web UI for server"                                                   ON)
-option(LLAMA_USE_PREBUILT_WEBUI     "llama: use prebuilt WebUI from HF Bucket when available (requires LLAMA_BUILD_WEBUI=ON)"       ON)
-option(LLAMA_TOOLS_INSTALL          "llama: install tools"                                                                          ${LLAMA_TOOLS_INSTALL_DEFAULT})
-option(LLAMA_TESTS_INSTALL          "llama: install tests"                                                                          ON)
+option(LLAMA_BUILD_TESTS    "llama: build tests"          ${LLAMA_STANDALONE})
+option(LLAMA_BUILD_TOOLS    "llama: build tools"          ${LLAMA_STANDALONE})
+option(LLAMA_BUILD_EXAMPLES "llama: build examples"       ${LLAMA_STANDALONE})
+option(LLAMA_BUILD_SERVER   "llama: build server example" ${LLAMA_STANDALONE})
+option(LLAMA_BUILD_WEBUI    "llama: build the embedded Web UI for server"  ON)
+option(LLAMA_TOOLS_INSTALL  "llama: install tools"        ${LLAMA_TOOLS_INSTALL_DEFAULT})
+option(LLAMA_TESTS_INSTALL  "llama: install tests"        ON)

 # 3rd party libs
 option(LLAMA_OPENSSL    "llama: use openssl to support HTTPS" ON)
@@ -46,9 +46,7 @@ Before submitting your PR:
    - provide KL divergence data calculated vs. the FP16/BF16 (whichever is the native precision) version for both the new type as well as types of similar size
    - provide [performance data](https://github.com/ggml-org/llama.cpp/tree/master/tools/llama-bench) for the new type in comparison to types of similar size on pure CPU
 - Consider allowing write access to your branch for faster reviews, as reviewers can push commits directly
- If you are a new contributor
-    - Limit your open PRs to 1
-    - Do not submit trivial fixes (e.g. typos, formatting changes)
+- If you are a new contributor, limit your open PRs to 1.

 After submitting your PR:
 - Expect requests for modifications to ensure the code meets llama.cpp's standards for quality and long-term maintainability
@@ -172,7 +172,6 @@ Instructions for adding support for new models: [HOWTO-add-model.md](docs/develo
 - JavaScript/Wasm (works in browser): [tangledgroup/llama-cpp-wasm](https://github.com/tangledgroup/llama-cpp-wasm)
 - Typescript/Wasm (nicer API, available on npm): [ngxson/wllama](https://github.com/ngxson/wllama)
 - Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb)
- Ruby: [docusealco/rllama](https://github.com/docusealco/rllama)
 - Rust (more features): [edgenai/llama_cpp-rs](https://github.com/edgenai/llama_cpp-rs)
 - Rust (nicer API): [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp)
 - Rust (more direct bindings): [utilityai/llama-cpp-rs](https://github.com/utilityai/llama-cpp-rs)
@@ -24,6 +24,6 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
-set(CMAKE_C_FLAGS "-march=rv64gcv_zfh_zvfh_zba_zicbop -mabi=lp64d -fno-tree-vectorize -fno-tree-loop-vectorize ${CMAKE_C_FLAGS}")
-set(CMAKE_CXX_FLAGS "-march=rv64gcv_zfh_zvfh_zba_zicbop -mabi=lp64d -fno-tree-vectorize -fno-tree-loop-vectorize ${CMAKE_CXX_FLAGS}")
+set(CMAKE_C_FLAGS "-march=rv64gcv_zfh_zba_zicbop -mabi=lp64d ${CMAKE_C_FLAGS}")
+set(CMAKE_CXX_FLAGS "-march=rv64gcv_zfh_zba_zicbop -mabi=lp64d ${CXX_FLAGS}")
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -latomic")
@@ -308,14 +308,12 @@ static bool common_params_handle_remote_preset(common_params & params, llama_exa
    common_download_opts opts;
    opts.bearer_token = params.hf_token;
    opts.offline = params.offline;
-
-    LOG_TRC("%s: looking for remote preset at %s\n", __func__, preset_url.c_str());
    const int status = common_download_file_single(preset_url, preset_path, opts);
    const bool has_preset = status >= 200 && status < 400;

    // remote preset is optional, so we don't error out if not found
    if (has_preset) {
-        LOG_TRC("%s: applying remote preset from %s\n", __func__, preset_url.c_str());
+        LOG_INF("applying remote preset from %s\n", preset_url.c_str());
        common_preset_context ctx(ex, /* only_remote_allowed */ true);
        common_preset global;
        auto remote_presets = ctx.load_from_ini(preset_path, global);
@@ -328,7 +326,7 @@ static bool common_params_handle_remote_preset(common_params & params, llama_exa
            throw std::runtime_error("Remote preset.ini does not contain [" + std::string(hf_tag) + "] section");
        }
    } else {
-        LOG_TRC("%s: no remote preset found, skipping\n", __func__);
+        LOG_INF("%s", "no remote preset found, skipping\n");
    }

    return has_preset;
@@ -359,7 +357,8 @@ static handle_model_result common_params_handle_model(struct common_params_model
        auto download_result = common_download_model(model, opts, true);

        if (download_result.model_path.empty()) {
-            throw std::runtime_error("failed to download model from Hugging Face");
+            LOG_ERR("error: failed to download model from Hugging Face\n");
+            exit(1);
        }

        model.name = model.hf_repo;
@@ -381,7 +380,8 @@ static handle_model_result common_params_handle_model(struct common_params_model
        opts.offline = offline;
        auto download_result = common_download_model(model, opts);
        if (download_result.model_path.empty()) {
-            throw std::runtime_error("failed to download model from " + model.url);
+            LOG_ERR("error: failed to download model from %s\n", model.url.c_str());
+            exit(1);
        }
    }

@@ -435,25 +435,6 @@ static bool parse_bool_value(const std::string & value) {
 // CLI argument parsing functions
 //

-void common_params_handle_models(common_params & params, llama_example curr_ex) {
-    auto res = common_params_handle_model(params.model, params.hf_token, params.offline);
-    if (params.no_mmproj) {
-        params.mmproj = {};
-    } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) {
-        // optionally, handle mmproj model when -hf is specified
-        params.mmproj = res.mmproj;
-    }
-    // only download mmproj if the current example is using it
-    for (const auto & ex : mmproj_examples) {
-        if (curr_ex == ex) {
-            common_params_handle_model(params.mmproj,    params.hf_token, params.offline);
-            break;
-        }
-    }
-    common_params_handle_model(params.speculative.draft.mparams, params.hf_token, params.offline);
-    common_params_handle_model(params.vocoder.model,             params.hf_token, params.offline);
-}
-
 static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {
    common_params & params = ctx_arg.params;

@@ -607,7 +588,22 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context

    // handle model and download
    if (!skip_model_download) {
-        common_params_handle_models(params, ctx_arg.ex);
+        auto res = common_params_handle_model(params.model, params.hf_token, params.offline);
+        if (params.no_mmproj) {
+            params.mmproj = {};
+        } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) {
+            // optionally, handle mmproj model when -hf is specified
+            params.mmproj = res.mmproj;
+        }
+        // only download mmproj if the current example is using it
+        for (const auto & ex : mmproj_examples) {
+            if (ctx_arg.ex == ex) {
+                common_params_handle_model(params.mmproj,    params.hf_token, params.offline);
+                break;
+            }
+        }
+        common_params_handle_model(params.speculative.draft.mparams, params.hf_token, params.offline);
+        common_params_handle_model(params.vocoder.model,             params.hf_token, params.offline);
    }

    // model is required (except for server)
@@ -2223,7 +2219,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
    if (llama_supports_rpc()) {
        add_opt(common_arg(
            {"--rpc"}, "SERVERS",
-            "comma-separated list of RPC servers (host:port)",
+            "comma separated list of RPC servers (host:port)",
            [](common_params & params, const std::string & value) {
                add_rpc_devices(value);
                GGML_UNUSED(params);
@@ -3303,20 +3299,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
    ).set_env("LLAMA_LOG_VERBOSITY"));
    add_opt(common_arg(
        {"--log-prefix"},
-        {"--no-log-prefix"},
        "Enable prefix in log messages",
-        [](common_params &, bool value) {
-            common_log_set_prefix(common_log_main(), value);
+        [](common_params &) {
+            common_log_set_prefix(common_log_main(), true);
        }
-    ).set_env("LLAMA_ARG_LOG_PREFIX"));
+    ).set_env("LLAMA_LOG_PREFIX"));
    add_opt(common_arg(
        {"--log-timestamps"},
-        {"--no-log-timestamps"},
        "Enable timestamps in log messages",
-        [](common_params &, bool value) {
-            common_log_set_timestamps(common_log_main(), value);
+        [](common_params &) {
+            common_log_set_timestamps(common_log_main(), true);
        }
-    ).set_env("LLAMA_ARG_LOG_TIMESTAMPS"));
+    ).set_env("LLAMA_LOG_TIMESTAMPS"));

    //
    // speculative parameters
@@ -3556,12 +3550,25 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
        }
    ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL"));
    add_opt(common_arg(
-        {"--spec-type"}, common_speculative_all_types_str(),
-        string_format("comma-separated list of types of speculative decoding to use (default: %s)\n",
-            common_speculative_type_name_str(params.speculative.types).c_str()),
+        {"--spec-type"}, "[none|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]",
+        string_format("type of speculative decoding to use when no draft model is provided (default: %s)\n",
+            common_speculative_type_to_str(params.speculative.type).c_str()),
        [](common_params & params, const std::string & value) {
-            const auto enabled_types = string_split<std::string>(value, ',');
-            params.speculative.types = common_speculative_types_from_names(enabled_types);
+            if (value == "none") {
+                params.speculative.type = COMMON_SPECULATIVE_TYPE_NONE;
+            } else if (value == "ngram-cache") {
+                params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_CACHE;
+            } else if (value == "ngram-simple") {
+                params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE;
+            } else if (value == "ngram-map-k") {
+                params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K;
+            } else if (value == "ngram-map-k4v") {
+                params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V;
+            } else if (value == "ngram-mod") {
+                params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_MOD;
+            } else {
+                throw std::invalid_argument("unknown speculative decoding type without draft model");
+            }
        }
    ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_TYPE"));
    add_opt(common_arg(
@@ -4050,7 +4057,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
        {"--spec-default"},
        string_format("enable default speculative decoding config"),
        [](common_params & params) {
-            params.speculative.types = { COMMON_SPECULATIVE_TYPE_NGRAM_MOD };
+            params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_MOD;
            params.speculative.ngram_mod.n_match = 24;
            params.speculative.ngram_mod.n_min = 48;
            params.speculative.ngram_mod.n_max = 64;
@@ -129,8 +129,5 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<com
 // see: https://github.com/ggml-org/llama.cpp/issues/18163
 void common_params_add_preset_options(std::vector<common_arg> & args);

-// Populate model paths (main model, mmproj, etc) from -hf if necessary
-void common_params_handle_models(common_params & params, llama_example curr_ex);
-
 // initialize argument parser context - used by test-arg-parser and preset
 common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);
@@ -369,7 +369,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
                                           arguments.name_suffix) +
                           arguments.value_prefix +
                           (schema_info.resolves_to_string(param_schema) ?
-                                p.tool_arg_string_value(until_suffix) :
+                                p.tool_arg_string_value(p.schema(until_suffix,
+                                                                 "tool-" + name + "-arg-" + param_name + "-schema",
+                                                                 param_schema, true)) :
                                p.tool_arg_json_value(p.schema(
                                    p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +
                                    p.space()) +
@@ -366,29 +366,15 @@ void common_init() {
    SetConsoleCP(CP_UTF8);
 #endif

-    common_log_set_prefix(common_log_main(), true);
-    common_log_set_timestamps(common_log_main(), true);
-
    llama_log_set(common_log_default_callback, NULL);
-}

-void common_params_print_info(const common_params & params) {
 #ifdef NDEBUG
    const char * build_type = "";
 #else
    const char * build_type = " (debug)";
 #endif
-    LOG_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type);

-    LOG_INF("log_info: verbosity = %d (adjust with the `-lv N` CLI arg)\n", common_log_get_verbosity_thold());
-    LOG_INF("device_info:\n");
-    for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
-        auto * dev = ggml_backend_dev_get(i);
-        size_t free, total;
-        ggml_backend_dev_memory(dev, &free, &total);
-        LOG_INF("  - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);
-    }
-    LOG_INF("%s\n", common_params_get_system_info(params).c_str());
+    LOG_DBG("build: %d (%s) with %s for %s%s\n", llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type);
 }

 std::string common_params_get_system_info(const common_params & params) {
@@ -1161,8 +1147,7 @@ common_init_result::common_init_result(common_params & params) :
    auto cparams = common_context_params_to_llama(params);

    if (params.fit_params) {
-        LOG_INF("%s: fitting params to device memory ...\n", __func__);
-        LOG_INF("%s: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n", __func__);
+        LOG_INF("%s: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on\n", __func__);
        common_fit_params(params.model.path.c_str(), &mparams, &cparams,
            params.tensor_split,
            params.tensor_buft_overrides.data(),
@@ -1211,7 +1196,7 @@ common_init_result::common_init_result(common_params & params) :
    // initialize once
    for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) {
        if (llama_vocab_is_eog(vocab, i)) {
-            LOG_TRC("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(vocab, i).c_str(), -INFINITY);
+            LOG_INF("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(vocab, i).c_str(), -INFINITY);
            params.sampling.logit_bias_eog.push_back({i, -INFINITY});
        }
    }
@@ -1224,12 +1209,12 @@ common_init_result::common_init_result(common_params & params) :
    }

    //if (params.sampling.penalty_last_n == -1) {
-    //    LOG_TRC("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
+    //    LOG_INF("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
    //    params.sampling.penalty_last_n = llama_n_ctx(lctx);
    //}

    //if (params.sampling.dry_penalty_last_n == -1) {
-    //    LOG_TRC("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
+    //    LOG_INF("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
    //    params.sampling.dry_penalty_last_n = llama_n_ctx(lctx);
    //}

@@ -1437,7 +1422,7 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {

    // try to remove the last tokens
    if (!llama_memory_seq_rm(mem, 0, 1, -1)) {
-        LOG_TRC("%s: the context does not support partial sequence removal\n", __func__);
+        LOG_WRN("%s: the context does not support partial sequence removal\n", __func__);
        res = COMMON_CONTEXT_SEQ_RM_TYPE_FULL;
        goto done;
    }
@@ -157,9 +157,9 @@ enum common_params_sampling_config : uint64_t {

 enum common_speculative_type {
    COMMON_SPECULATIVE_TYPE_NONE,          // no speculative decoding
-    COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE,  // standalone draft model speculative decoding
-    COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3,  // Eagle3 speculative decoding
-    COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE,  // simple self-speculative decoding based on n-grams
+    COMMON_SPECULATIVE_TYPE_DRAFT,         // draft model
+    COMMON_SPECULATIVE_TYPE_EAGLE3,        // eagle draft model
+    COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE,  // simple self-speculative decoding
    COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K,   // self-speculative decoding with n-gram keys only
    COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
    COMMON_SPECULATIVE_TYPE_NGRAM_MOD,
@@ -295,6 +295,8 @@ struct common_params_model {
    std::string name        = ""; // in format <user>/<model>[:<tag>] (tag is optional)     // NOLINT
 };

+struct common_ngram_mod;
+
 // draft-model-based speculative decoding parameters
 struct common_params_speculative_draft {
    int32_t n_max = 16; // maximum number of tokens to draft during speculative decoding
@@ -326,6 +328,9 @@ struct common_params_speculative_ngram_mod {

    int32_t n_max = 64;
    int32_t n_min = 48;
+
+    // shared instance of the ngram container for all speculative decoding contexts
+    std::shared_ptr<common_ngram_mod> obj;
 };

 struct common_params_speculative_ngram_map {
@@ -340,9 +345,9 @@ struct common_params_speculative_ngram_cache {
 };

 struct common_params_speculative {
-    std::vector<enum common_speculative_type> types = { COMMON_SPECULATIVE_TYPE_NONE };
+    // TODO: become a vector in order to support "chains of speculators"
+    common_speculative_type type = COMMON_SPECULATIVE_TYPE_NONE;

-    // used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft model
    common_params_speculative_draft draft;

    common_params_speculative_ngram_mod ngram_mod;
@@ -605,11 +610,7 @@ struct common_params {
    std::map<std::string, std::string> default_template_kwargs;

    // webui configs
-#ifdef LLAMA_WEBUI_DEFAULT_ENABLED
-    bool webui = LLAMA_WEBUI_DEFAULT_ENABLED != 0;
-#else
-    bool webui = true; // default to enabled when not set
-#endif
+    bool webui = true;
    bool webui_mcp_proxy = false;
    std::string webui_config_json;

@@ -690,7 +691,6 @@ struct common_params {
 // initializes the logging system and prints info about the build
 void common_init();

-void common_params_print_info(const common_params & params);
 std::string common_params_get_system_info(const common_params & params);

 bool parse_cpu_range(const std::string & range, bool(&boolmask)[GGML_MAX_N_THREADS]);
@@ -320,9 +320,9 @@ static int common_download_file_single_online(const std::string & url,

    auto head = cli.Head(parts.path);
    if (!head || head->status < 200 || head->status >= 300) {
-        LOG_TRC("%s: HEAD failed, status: %d\n", __func__, head ? head->status : -1);
+        LOG_WRN("%s: HEAD failed, status: %d\n", __func__, head ? head->status : -1);
        if (file_exists) {
-            LOG_TRC("%s: using cached file (HEAD failed): %s\n", __func__, path.c_str());
+            LOG_INF("%s: using cached file (HEAD failed): %s\n", __func__, path.c_str());
            return 304; // 304 Not Modified - fake cached response
        }
        return head ? head->status : -1;
@@ -168,7 +168,7 @@ static void common_params_fit_impl(

    // step 1: get data for default parameters and check whether any changes are necessary in the first place

-    LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
+    LOG_INF("%s: getting device memory data for initial parameters:\n", __func__);
    const dmds_t dmds_full = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
    const size_t nd = devs.size(); // number of devices

@@ -213,13 +213,13 @@ static void common_params_fit_impl(
        LOG_INF("%s: projected to use %" PRId64 " MiB of host memory vs. %" PRId64 " MiB of total host memory\n",
            __func__, sum_projected_used/MiB, sum_free/MiB);
        if (sum_projected_free >= margins[0]) {
-            LOG_TRC("%s: will leave %" PRId64 " >= %" PRId64 " MiB of system memory, no changes needed\n",
+            LOG_INF("%s: will leave %" PRId64 " >= %" PRId64 " MiB of system memory, no changes needed\n",
                __func__, sum_projected_free/MiB, margins[0]/MiB);
            return;
        }
    } else {
        if (nd > 1) {
-            LOG_TRC("%s: projected memory use with initial parameters [MiB]:\n", __func__);
+            LOG_INF("%s: projected memory use with initial parameters [MiB]:\n", __func__);
        }
        for (size_t id = 0; id < nd; id++) {
            const llama_device_memory_data & dmd = dmds_full[id];
@@ -234,16 +234,16 @@ static void common_params_fit_impl(
            sum_projected_model += dmd.mb.model;

            if (nd > 1) {
-                LOG_TRC("%s:   - %s: %6" PRId64 " total, %6" PRId64 " used, %6" PRId64 " free vs. target of %6" PRId64 "\n",
+                LOG_INF("%s:   - %s: %6" PRId64 " total, %6" PRId64 " used, %6" PRId64 " free vs. target of %6" PRId64 "\n",
                    __func__, dev_names[id].c_str(), dmd.total/MiB, projected_used/MiB, projected_free/MiB, margins[id]/MiB);
            }
        }
        assert(sum_free >= 0 && sum_projected_used >= 0);
-        LOG_TRC("%s: projected to use %" PRId64 " MiB of device memory vs. %" PRId64 " MiB of free device memory\n",
+        LOG_INF("%s: projected to use %" PRId64 " MiB of device memory vs. %" PRId64 " MiB of free device memory\n",
            __func__, sum_projected_used/MiB, sum_free/MiB);
        if (nd == 1) {
            if (projected_free_per_device[0] >= margins[0]) {
-                LOG_TRC("%s: will leave %" PRId64 " >= %" PRId64 " MiB of free device memory, no changes needed\n",
+                LOG_INF("%s: will leave %" PRId64 " >= %" PRId64 " MiB of free device memory, no changes needed\n",
                    __func__, projected_free_per_device[0]/MiB, margins[0]/MiB);
                return;
            }
@@ -256,7 +256,7 @@ static void common_params_fit_impl(
                }
            }
            if (!changes_needed) {
-                LOG_TRC("%s: targets for free memory can be met on all devices, no changes needed\n", __func__);
+                LOG_INF("%s: targets for free memory can be met on all devices, no changes needed\n", __func__);
                return;
            }
        }
@@ -275,10 +275,10 @@ static void common_params_fit_impl(
        }
        if (global_surplus < 0) {
            if (nd <= 1) {
-                LOG_TRC("%s: cannot meet free memory target of %" PRId64 " MiB, need to reduce device memory by %" PRId64 " MiB\n",
+                LOG_INF("%s: cannot meet free memory target of %" PRId64 " MiB, need to reduce device memory by %" PRId64 " MiB\n",
                    __func__, margins[0]/MiB, -global_surplus/MiB);
            } else {
-                LOG_TRC(
+                LOG_INF(
                    "%s: cannot meet free memory targets on all devices, need to use %" PRId64 " MiB less in total\n",
                    __func__, -global_surplus/MiB);
            }
@@ -320,28 +320,28 @@ static void common_params_fit_impl(

                        const int64_t bytes_per_ctx = (sum_projected_used - sum_projected_used_min_ctx) / (hp_nct - n_ctx_min);
                        const int64_t memory_reduction = (hp_nct - cparams->n_ctx) * bytes_per_ctx;
-                        LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",
+                        LOG_INF("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",
                            __func__, hp_nct, cparams->n_ctx, memory_reduction/MiB);
                        if (nd <= 1) {
-                            LOG_TRC("%s: entire model can be fit by reducing context\n", __func__);
+                            LOG_INF("%s: entire model can be fit by reducing context\n", __func__);
                            return;
                        }
-                        LOG_TRC("%s: entire model should be fit across devices by reducing context\n", __func__);
+                        LOG_INF("%s: entire model should be fit across devices by reducing context\n", __func__);
                    } else {
                        const int64_t memory_reduction = sum_projected_used - sum_projected_used_min_ctx;
-                        LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",
+                        LOG_INF("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",
                            __func__, hp_nct, cparams->n_ctx, memory_reduction/MiB);
                    }
                } else {
                    if (n_ctx_min == UINT32_MAX) {
-                        LOG_TRC("%s: user has requested full context size of %" PRIu32 " -> no change\n", __func__, hp_nct);
+                        LOG_INF("%s: user has requested full context size of %" PRIu32 " -> no change\n", __func__, hp_nct);
                    } else {
-                        LOG_TRC("%s: default model context size is %" PRIu32 " which is <= the min. context size of %" PRIu32 " -> no change\n",
+                        LOG_INF("%s: default model context size is %" PRIu32 " which is <= the min. context size of %" PRIu32 " -> no change\n",
                            __func__, hp_nct, n_ctx_min);
                    }
                }
            } else {
-                LOG_TRC("%s: context size set by user to %" PRIu32 " -> no change\n", __func__, cparams->n_ctx);
+                LOG_INF("%s: context size set by user to %" PRIu32 " -> no change\n", __func__, cparams->n_ctx);
            }
        }
    }
@@ -485,10 +485,10 @@ static void common_params_fit_impl(
        const dmds_t dmd_nl = common_get_device_memory_data(
            path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);

-        LOG_TRC("%s: memory for test allocation by device:\n", func_name);
+        LOG_INF("%s: memory for test allocation by device:\n", func_name);
        for (size_t id = 0; id < nd; id++) {
            const ngl_t & n = ngl_per_device[id];
-            LOG_TRC(
+            LOG_INF(
                "%s: id=%zu, n_layer=%2" PRIu32 ", n_part=%2" PRIu32 ", overflow_type=%d, mem=%6" PRId64 " MiB\n",
                func_name, id, n.n_layer, n.n_part, int(n.overflow_type), dmd_nl[id].mb.total()/MiB);
        }
@@ -509,7 +509,7 @@ static void common_params_fit_impl(
        tensor_buft_overrides[1] = {nullptr, nullptr};
        mparams->tensor_buft_overrides = tensor_buft_overrides;

-        LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
+        LOG_INF("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
        const dmds_t dmds_cpu_moe = common_get_device_memory_data(
            path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);

@@ -519,10 +519,10 @@ static void common_params_fit_impl(
        }

        if (global_surplus_cpu_moe > 0) {
-            LOG_TRC("%s: with only dense weights in device memory there is a total surplus of %" PRId64 " MiB\n",
+            LOG_INF("%s: with only dense weights in device memory there is a total surplus of %" PRId64 " MiB\n",
                __func__, global_surplus_cpu_moe/MiB);
        } else {
-            LOG_TRC("%s: with only dense weights in device memory there is still a total deficit of %" PRId64 " MiB\n",
+            LOG_INF("%s: with only dense weights in device memory there is still a total deficit of %" PRId64 " MiB\n",
                __func__, -global_surplus_cpu_moe/MiB);
        }

@@ -535,7 +535,7 @@ static void common_params_fit_impl(
    targets.reserve(nd);
    for (size_t id = 0; id < nd; id++) {
        targets.push_back(dmds_full[id].free - margins[id]);
-        LOG_TRC("%s: id=%zu, target=%" PRId64 " MiB\n", __func__, id, targets[id]/MiB);
+        LOG_INF("%s: id=%zu, target=%" PRId64 " MiB\n", __func__, id, targets[id]/MiB);
    }

    std::vector<ggml_backend_buffer_type_t> overflow_bufts; // which bufts the first partial layer of a device overflows to:
@@ -555,9 +555,9 @@ static void common_params_fit_impl(
    //   - once we only have a difference of a single layer, stop and return the lower bound that just barely still fits
    //   - the last device has the output layer, which cannot be a partial layer
    if (hp_nex == 0) {
-        LOG_TRC("%s: filling dense layers back-to-front:\n", __func__);
+        LOG_INF("%s: filling dense layers back-to-front:\n", __func__);
    } else {
-        LOG_TRC("%s: filling dense-only layers back-to-front:\n", __func__);
+        LOG_INF("%s: filling dense-only layers back-to-front:\n", __func__);
    }
    for (int id = nd - 1; id >= 0; id--) {
        uint32_t n_unassigned = hp_ngl + 1;
@@ -576,7 +576,7 @@ static void common_params_fit_impl(
            if (mem_high[id] > targets[id]) {
                assert(ngl_per_device_high[id].n_layer > ngl_per_device[id].n_layer);
                uint32_t delta = ngl_per_device_high[id].n_layer - ngl_per_device[id].n_layer;
-                LOG_TRC("%s: start filling device %" PRIu32 ", delta=%" PRIu32 "\n", __func__, id, delta);
+                LOG_INF("%s: start filling device %" PRIu32 ", delta=%" PRIu32 "\n", __func__, id, delta);
                while (delta > 1) {
                    uint32_t step_size = int64_t(delta) * (targets[id] - mem[id]) / (mem_high[id] - mem[id]);
                    step_size = std::max(step_size, uint32_t(1));
@@ -593,11 +593,11 @@ static void common_params_fit_impl(
                    if (mem_test[id] <= targets[id]) {
                        ngl_per_device = ngl_per_device_test;
                        mem            = mem_test;
-                        LOG_TRC("%s: set ngl_per_device[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device[id].n_layer);
+                        LOG_INF("%s: set ngl_per_device[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device[id].n_layer);
                    } else {
                        ngl_per_device_high = ngl_per_device_test;
                        mem_high            = mem_test;
-                        LOG_TRC("%s: set ngl_per_device_high[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device_high[id].n_layer);
+                        LOG_INF("%s: set ngl_per_device_high[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device_high[id].n_layer);
                    }
                    delta = ngl_per_device_high[id].n_layer - ngl_per_device[id].n_layer;
                }
@@ -605,12 +605,12 @@ static void common_params_fit_impl(
                assert(ngl_per_device_high[id].n_layer == n_unassigned);
                ngl_per_device = ngl_per_device_high;
                mem            = mem_high;
-                LOG_TRC("%s: set ngl_per_device[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device[id].n_layer);
+                LOG_INF("%s: set ngl_per_device[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device[id].n_layer);
            }
        }

        const int64_t projected_margin = dmds_full[id].free - mem[id];
-        LOG_TRC(
+        LOG_INF(
            "%s:   - %s: %2" PRIu32 " layers, %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",
            __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, mem[id]/MiB, projected_margin/MiB);
    }
@@ -634,7 +634,7 @@ static void common_params_fit_impl(
    }
    assert(id_dense_start < nd);

-    LOG_TRC("%s: converting dense-only layers to full layers and filling them front-to-back with overflow to next device/system memory:\n", __func__);
+    LOG_INF("%s: converting dense-only layers to full layers and filling them front-to-back with overflow to next device/system memory:\n", __func__);
    for (size_t id = 0; id <= id_dense_start && id_dense_start < nd; id++) {
        std::vector<ngl_t> ngl_per_device_high = ngl_per_device;
        for (size_t jd = id_dense_start; jd < nd; jd++) {
@@ -674,13 +674,13 @@ static void common_params_fit_impl(
                    ngl_per_device = ngl_per_device_test;
                    mem            = mem_test;
                    id_dense_start = id_dense_start_test;
-                    LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start=%zu\n",
+                    LOG_INF("%s: set ngl_per_device[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start=%zu\n",
                        __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
                } else {
                    ngl_per_device_high = ngl_per_device_test;
                    mem_high            = mem_test;
                    id_dense_start_high = id_dense_start_test;
-                    LOG_TRC("%s: set ngl_per_device_high[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start_high=%zu\n",
+                    LOG_INF("%s: set ngl_per_device_high[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start_high=%zu\n",
                        __func__, id, ngl_per_device_high[id].n_layer, ngl_per_device_high[id].n_part, id_dense_start_high);
                }
                assert(ngl_per_device_high[id].n_full() >= ngl_per_device[id].n_full());
@@ -690,7 +690,7 @@ static void common_params_fit_impl(
            ngl_per_device = ngl_per_device_high;
            mem            = mem_high;
            id_dense_start = id_dense_start_high;
-            LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start=%zu\n",
+            LOG_INF("%s: set ngl_per_device[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start=%zu\n",
                __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
        }

@@ -710,44 +710,44 @@ static void common_params_fit_impl(
            if (id < nd - 1) {
                overflow_bufts_test[id] = ggml_backend_dev_buffer_type(devs[id + 1]);
            }
-            LOG_TRC("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_UP\n", __func__);
+            LOG_INF("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_UP\n", __func__);
            std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);
            if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {
                ngl_per_device = ngl_per_device_test;
                overflow_bufts = overflow_bufts_test;
                mem            = mem_test;
                id_dense_start = id_dense_start_test;
-                LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", UP), id_dense_start=%zu\n",
+                LOG_INF("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", UP), id_dense_start=%zu\n",
                    __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);

                ngl_per_device_test[id].overflow_type = LAYER_FRACTION_GATE;
-                LOG_TRC("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_GATE\n", __func__);
+                LOG_INF("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_GATE\n", __func__);
                mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);
                if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {
                    ngl_per_device = ngl_per_device_test;
                    overflow_bufts = overflow_bufts_test;
                    mem            = mem_test;
                    id_dense_start = id_dense_start_test;
-                    LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", GATE), id_dense_start=%zu\n",
+                    LOG_INF("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", GATE), id_dense_start=%zu\n",
                        __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
                }
            } else {
                ngl_per_device_test[id].overflow_type = LAYER_FRACTION_ATTN;
-                LOG_TRC("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_ATTN\n", __func__);
+                LOG_INF("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_ATTN\n", __func__);
                mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);
                if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {
                    ngl_per_device = ngl_per_device_test;
                    overflow_bufts = overflow_bufts_test;
                    mem            = mem_test;
                    id_dense_start = id_dense_start_test;
-                    LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", ATTN), id_dense_start=%zu\n",
+                    LOG_INF("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", ATTN), id_dense_start=%zu\n",
                        __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
                }
            }
        }

        const int64_t projected_margin = dmds_full[id].free - mem[id];
-        LOG_TRC(
+        LOG_INF(
            "%s:   - %s: %2" PRIu32 " layers (%2" PRIu32 " overflowing), %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",
            __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, ngl_per_device[id].n_part, mem[id]/MiB, projected_margin/MiB);
    }
@@ -755,7 +755,7 @@ static void common_params_fit_impl(
    // print info for devices that were not changed during the conversion from dense only to full layers:
    for (size_t id = id_dense_start + 1; id < nd; id++) {
        const int64_t projected_margin = dmds_full[id].free - mem[id];
-        LOG_TRC(
+        LOG_INF(
            "%s:   - %s: %2" PRIu32 " layers (%2" PRIu32 " overflowing), %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",
            __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, ngl_per_device[id].n_part, mem[id]/MiB, projected_margin/MiB);
    }
@@ -776,7 +776,7 @@ enum common_params_fit_status common_fit_params(
    common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS;
    try {
        common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, log_level);
-        LOG_TRC("%s: successfully fit params to free device memory\n", __func__);
+        LOG_INF("%s: successfully fit params to free device memory\n", __func__);
    } catch (const common_params_fit_exception & e) {
        LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());
        status = COMMON_PARAMS_FIT_STATUS_FAILURE;
@@ -785,7 +785,7 @@ enum common_params_fit_status common_fit_params(
        status = COMMON_PARAMS_FIT_STATUS_ERROR;
    }
    const int64_t t1_us = llama_time_us();
-    LOG_TRC("%s: fitting params to free memory took %.2f seconds\n", __func__, (t1_us - t0_us) * 1e-6);
+    LOG_INF("%s: fitting params to free memory took %.2f seconds\n", __func__, (t1_us - t0_us) * 1e-6);
    return status;
 }

@@ -925,7 +925,7 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
        }
    }
    for (const auto & td : table_data) {
-        LOG_TRC(td[0].c_str(),
+        LOG_INF(td[0].c_str(),
            __func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(),
            td[6].c_str(), td[7].c_str(), td[8].c_str());
    }
@@ -435,10 +435,10 @@ void common_log_flush(struct common_log * log) {
 static int common_get_verbosity(enum ggml_log_level level) {
    switch (level) {
        case GGML_LOG_LEVEL_DEBUG: return LOG_LEVEL_DEBUG;
-        case GGML_LOG_LEVEL_INFO:  return LOG_LEVEL_TRACE;
+        case GGML_LOG_LEVEL_INFO:  return LOG_LEVEL_INFO;
        case GGML_LOG_LEVEL_WARN:  return LOG_LEVEL_WARN;
        case GGML_LOG_LEVEL_ERROR: return LOG_LEVEL_ERROR;
-        case GGML_LOG_LEVEL_CONT:  return LOG_LEVEL_TRACE;
+        case GGML_LOG_LEVEL_CONT:  return LOG_LEVEL_INFO; // same as INFO
        case GGML_LOG_LEVEL_NONE:
        default:
            return LOG_LEVEL_OUTPUT;
@@ -21,8 +21,7 @@
 #    define LOG_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
 #endif

-#define LOG_LEVEL_DEBUG  5
-#define LOG_LEVEL_TRACE  4
+#define LOG_LEVEL_DEBUG  4
 #define LOG_LEVEL_INFO   3
 #define LOG_LEVEL_WARN   2
 #define LOG_LEVEL_ERROR  1
@@ -112,15 +111,13 @@ void common_log_flush         (struct common_log * log);                    // f
 #define LOGV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_NONE, verbosity,        __VA_ARGS__)

 #define LOG_DBG(...) LOG_TMPL(GGML_LOG_LEVEL_DEBUG, LOG_LEVEL_DEBUG,  __VA_ARGS__)
-#define LOG_TRC(...) LOG_TMPL(GGML_LOG_LEVEL_INFO,  LOG_LEVEL_TRACE,  __VA_ARGS__)
 #define LOG_INF(...) LOG_TMPL(GGML_LOG_LEVEL_INFO,  LOG_LEVEL_INFO,   __VA_ARGS__)
 #define LOG_WRN(...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  LOG_LEVEL_WARN,   __VA_ARGS__)
 #define LOG_ERR(...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, LOG_LEVEL_ERROR,  __VA_ARGS__)
 #define LOG_CNT(...) LOG_TMPL(GGML_LOG_LEVEL_CONT,  LOG_LEVEL_INFO,   __VA_ARGS__) // same as INFO

-#define LOG_DBGV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_DEBUG, verbosity, __VA_ARGS__)
-#define LOG_TRCV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_TRACE, verbosity, __VA_ARGS__)
 #define LOG_INFV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_INFO,  verbosity, __VA_ARGS__)
 #define LOG_WRNV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  verbosity, __VA_ARGS__)
 #define LOG_ERRV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, verbosity, __VA_ARGS__)
+#define LOG_DBGV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_DEBUG, verbosity, __VA_ARGS__)
 #define LOG_CNTV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_CONT,  verbosity, __VA_ARGS__)
@@ -163,13 +163,8 @@ void common_preset::merge(const common_preset & other) {
    }
 }

-void common_preset::apply_to_params(common_params & params, const std::set<std::string> & handled_keys) const {
+void common_preset::apply_to_params(common_params & params) const {
    for (const auto & [opt, val] : options) {
-        if (!handled_keys.empty()) {
-            if (!opt.env || handled_keys.find(opt.env) == handled_keys.end()) {
-                continue;
-            }
-        }
        // apply each option to params
        if (opt.handler_string) {
            opt.handler_string(params, val);
@@ -43,8 +43,7 @@ struct common_preset {
    void merge(const common_preset & other);

    // apply preset options to common_params
-    // optionally specify handled_keys to only apply a subset of options (identified by their env), if empty, apply all options
-    void apply_to_params(common_params & params, const std::set<std::string> & handled_keys = std::set<std::string>()) const;
+    void apply_to_params(common_params & params) const;
 };

 // interface for multiple presets in one file
@@ -158,6 +158,8 @@ static void common_reasoning_budget_apply(struct llama_sampler * smpl, llama_tok
    for (size_t i = 0; i < cur_p->size; i++) {
        if (cur_p->data[i].id != forced) {
            cur_p->data[i].logit = -INFINITY;
+        } else {
+            cur_p->data[i].logit = +INFINITY; // force the token
        }
    }
 }
@@ -171,12 +173,22 @@ static void common_reasoning_budget_reset(struct llama_sampler * smpl) {
    ctx->force_pos = 0;
 }

+// forward declaration for use in clone
 static struct llama_sampler * common_reasoning_budget_init_state(
        const struct llama_vocab * vocab, const std::vector<llama_token> & start_tokens,
        const std::vector<llama_token> & end_tokens, const std::vector<llama_token> & forced_tokens,
        int32_t budget, common_reasoning_budget_state initial_state);

-static struct llama_sampler * common_reasoning_budget_clone(const struct llama_sampler * smpl);
+static struct llama_sampler * common_reasoning_budget_clone(const struct llama_sampler * smpl) {
+    const auto * ctx = (const common_reasoning_budget_ctx *) smpl->ctx;
+    return common_reasoning_budget_init_state(
+        ctx->vocab,
+        ctx->start_matcher.tokens,
+        ctx->end_matcher.tokens,
+        ctx->forced_tokens,
+        ctx->budget,
+        ctx->state);
+}

 static void common_reasoning_budget_free(struct llama_sampler * smpl) {
    delete (common_reasoning_budget_ctx *) smpl->ctx;
@@ -195,15 +207,6 @@ static struct llama_sampler_i common_reasoning_budget_i = {
    /* .backend_set_input = */ nullptr,
 };

-static struct llama_sampler * common_reasoning_budget_clone(const struct llama_sampler * smpl) {
-    const auto * ctx = (const common_reasoning_budget_ctx *) smpl->ctx;
-
-    return llama_sampler_init(
-        /* .iface = */ &common_reasoning_budget_i,
-        /* .ctx   = */ new common_reasoning_budget_ctx(*ctx)
-    );
-}
-
 static struct llama_sampler * common_reasoning_budget_init_state(
        const struct llama_vocab             * vocab,
        const std::vector<llama_token>       & start_tokens,
@@ -547,8 +547,6 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co
    auto & chain = gsmpl->chain;
    auto & cur_p = gsmpl->cur_p; // initialized by set_logits

-    gsmpl->set_logits(ctx, idx);
-
    // Check if a backend sampler has already sampled a token in which case we
    // return that token id directly.
    {
@@ -560,17 +558,17 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co
            GGML_ASSERT(!gsmpl->grmr    && "using grammar in combination with backend sampling is not supported");
            GGML_ASSERT(!gsmpl->rbudget && "using reasoning budget in combination with backend sampling is not supported");

-            for (size_t i = 0; i < cur_p.size; ++i) {
-                if (cur_p.data[i].id == id) {
-                    cur_p.selected = i;
-                    break;
-                }
-            }
+            // TODO: simplify
+            gsmpl->cur.resize(1);
+            gsmpl->cur[0] = { id, 0.0f, 1.0f };
+            cur_p = { gsmpl->cur.data(), gsmpl->cur.size(), 0, true };

            return id;
        }
    }

+    gsmpl->set_logits(ctx, idx);
+
    // apply reasoning budget first
    llama_sampler_apply(rbudget, &cur_p);

@@ -5,14 +5,8 @@

 struct common_speculative;

-// comma separated list the provided types
-std::string common_speculative_type_name_str(const std::vector<enum common_speculative_type> & types);
-
 // comma separated list of all types
-const char * common_speculative_all_types_str();
-
-// parse user provided types
-std::vector<enum common_speculative_type> common_speculative_types_from_names(const std::vector<std::string> & names);
+std::string common_speculative_type_name_str();

 // convert string to type
 enum common_speculative_type common_speculative_type_from_name(const std::string & name);
@@ -20,44 +14,26 @@ enum common_speculative_type common_speculative_type_from_name(const std::string
 // convert type to string
 std::string common_speculative_type_to_str(enum common_speculative_type type);

-common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq);
+common_speculative * common_speculative_init(common_params_speculative & params, llama_seq_id seq_id);

 void common_speculative_free(common_speculative * spec);

-struct common_speculative_draft_params {
-    // this flag is used to chain the drafts through all the available implementations
-    // after the first successful draft from an implementation, we set it
-    //   to false to prevent further drafts for that sequence
-    // at the end of the draft() call, all drafting flags will be reset to false
-    bool drafting = false;
-
-    // overrides individual configurations (-1 disabled)
-    // can be used to constraint the max draft based on the remaining context size
-    int32_t n_max = -1;
-
-    llama_pos   n_past;
-    llama_token id_last;
-
-    // TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls
-    const llama_tokens * prompt;
-
-    // the generated draft from the last _draft() call
-    llama_tokens * result;
-};
-
-common_speculative_draft_params & common_speculative_get_draft_params(common_speculative * spec, llama_seq_id seq_id);
-
 // optionally call once at the beginning of a new generation
-void common_speculative_begin(common_speculative * spec, llama_seq_id seq_id, const llama_tokens & prompt);
+void common_speculative_begin(common_speculative * spec, const llama_tokens & prompt);

-// process the batch and update the internal state of the speculative context
-bool common_speculative_process(common_speculative * spec, const llama_batch & batch);
+// sample up to n_draft tokens and add them to the batch using the draft model
+llama_tokens common_speculative_draft(
+                     common_speculative * spec,
+        const common_params_speculative & params,
+                     const llama_tokens & prompt,
+                              llama_pos   n_past,
+                            llama_token   id_last);

-// generate drafts for the sequences specified with `common_speculative_get_draft_params`
-void common_speculative_draft(common_speculative * spec);
+// informs the speculative decoder that n_accepted tokens were accepted by the target model
+void common_speculative_accept(common_speculative * spec, uint16_t n_accepted);

-// informs the speculative context that n_accepted tokens were accepted by the target model
-void common_speculative_accept(common_speculative * spec, llama_seq_id, uint16_t n_accepted);
+int32_t common_speculative_n_max(const common_speculative * spec, const common_params_speculative & params);
+int32_t common_speculative_n_min(const common_speculative * spec, const common_params_speculative & params);

 // print statistics about the speculative decoding
 void common_speculative_print_stats(const common_speculative * spec);
@@ -710,7 +710,7 @@ class ModelBase:
                self._repack_nvfp4(name, weight, scale, scale2, input_scale)

        # Flush any remaining experts (fallback if n_experts was unknown)
-        for bid, proj_type in list(expert_blocks.keys()):
+        for bid, proj_type in expert_blocks.keys():
            self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type)

        # Remove consumed tensors so get_tensors/modify_tensors won't see them
@@ -718,7 +718,7 @@ class ModelBase:
            self.model_tensors.pop(name, None)

        # Remove any remaining unused auxiliary tensors
-        for name in list(self.model_tensors.keys()):
+        for name in self.model_tensors.keys():
            if name.endswith((".k_scale", ".v_scale")):
                del self.model_tensors[name]

@@ -1570,9 +1570,6 @@ class TextModel(ModelBase):
        if chkhsh == "862f827721df956049dff5ca81a57f29e575280bc622e290d3bf4e35eca29015":
            # ref: https://huggingface.co/codefuse-ai/F2LLM-v2-4B
            res = "f2llmv2"
-        if chkhsh == "62f6fb0a6fd5098caeabb19b07a5c1099cafc8b9c40eab6ea89ece4ec02fbc57":
-            # ref: https://huggingface.co/sarvamai/sarvam-30b
-            res = "sarvam-moe"

        if res is None:
            logger.warning("\n")
@@ -2176,8 +2173,7 @@ class MmprojModel(ModelBase):
            text_config = {
                k: v for k, v in self.hparams.items() if k not in ["vision_encoder", "audio_encoder"]
            }
-            # mistral native params.json: "dim" is the text hidden size ("hidden_dim" is the FFN intermediate size)
-            self.n_embd_text = text_config.get("dim", 0)
+            self.n_embd_text = text_config.get("hidden_dim", 0)

        assert self.n_embd_text > 0, "n_embd not found in hparams"

@@ -2865,12 +2861,8 @@ class LlamaModel(TextModel):
        # fix for SmolVLM2, missing `num_attention_heads` in config.json
        if self.hf_arch == "VLlama3ForCausalLM":
            self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 32)
-        # Mistral consolidated format has no config.json; origin_hf_arch is HF-only.
-        if self.is_mistral_format:
-            self.origin_hf_arch = None
-        else:
-            hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
-            self.origin_hf_arch = hparams.get('architectures', [None])[0]
+        hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
+        self.origin_hf_arch = hparams.get('architectures', [None])[0]

    def set_vocab(self):
        if self.origin_hf_arch == "GlmasrModel":
@@ -3142,11 +3134,6 @@ class LlavaVisionModel(MmprojModel):
            assert self.hparams["norm_eps"] is not None, "norm_eps not found in params.json"
            if self.use_break_tok:
                self.img_break_tok_id = self.find_vparam(["image_break_token_id"])
-
-                # params.json may ship -1 placeholders (Mistral Medium 3.5)
-                # resolve the real id from the bundled tokenizer in that case
-                if self.img_break_tok_id < 0:
-                    self.img_break_tok_id = self.get_mistral_token_id("[IMG_BREAK]")
        else:
            raise ValueError(f"Unsupported model type: {self.hparams['model_type']}")
        logger.info(f"Image break token id: {self.img_break_tok_id}")
@@ -3166,24 +3153,6 @@ class LlavaVisionModel(MmprojModel):
                    return int(token_data["id"])
        raise ValueError(f"Token '{token}' not found in tokenizer config.")

-    def get_mistral_token_id(self, token: str) -> int:
-        # mistral native format ships tekken.json or a versioned spm tokenizer
-        tekken_file = self.dir_model / "tekken.json"
-        if tekken_file.is_file():
-            with open(tekken_file, "r", encoding="utf-8") as f:
-                data = json.load(f)
-            for entry in data.get("special_tokens", []):
-                if entry.get("token_str") == token:
-                    return int(entry["rank"])
-        tokenizer_json_file = self.dir_model / "tokenizer.json"
-        if tokenizer_json_file.is_file():
-            with open(tokenizer_json_file, "r", encoding="utf-8") as f:
-                data = json.load(f)
-            for entry in data.get("added_tokens", []):
-                if entry.get("content") == token:
-                    return int(entry["id"])
-        raise ValueError(f"Token '{token}' not found in mistral tokenizer files.")
-
    def set_gguf_parameters(self):
        super().set_gguf_parameters()
        hparams = self.hparams
@@ -8019,37 +7988,13 @@ class Gemma4Model(Gemma3Model):
        rope_freqs_full = torch.tensor(values, dtype=torch.float32)
        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs_full)

-    def _generate_nvfp4_tensors(self):
-        # Gemma-4 stores a per-layer router.per_expert_scale ([n_expert]) that scales
-        # each expert's contribution. It's mathematically equivalent to a per-expert
-        # scalar on the down_proj output, which is exactly where ffn_down_exps_s is
-        # applied at inference. Fold it into each expert's NVFP4 weight_scale_2 so the
-        # existing NVFP4 path produces the right scales.
-        n_experts = self.find_hparam(["num_local_experts", "num_experts"], optional=True) or 0
-        for name in [n for n in self.model_tensors if n.endswith(".router.per_expert_scale")]:
-            bid_match = re.search(r"\.layers\.(\d+)\.", name)
-            if bid_match is None:
-                continue
-            bid = bid_match.group(1)
-            prefix = name[: name.index(f".layers.{bid}.") + len(f".layers.{bid}.")]
-            w2_targets = [f"{prefix}experts.{e}.down_proj.weight_scale_2" for e in range(n_experts)]
-            present = [w2 in self.model_tensors for w2 in w2_targets]
-            if not any(present):
-                continue
-            assert all(present), f"layer {bid}: partial NVFP4 quantization across experts"
-            r = self.model_tensors.pop(name)
-            for e, w2 in enumerate(w2_targets):
-                s = self.model_tensors[w2]
-                self.model_tensors[w2] = lambda s=s, r=r, i=e: s() * r()[i]
-        super()._generate_nvfp4_tensors()
-
    @classmethod
    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
        name, gen = item

        if name.endswith("per_dim_scale") or name.endswith("layer_scalar"):
            name = name + ".weight"
-        if ".experts." in name and not name.endswith((".weight", ".weight_scale", ".weight_scale_2", ".input_scale")):
+        if ".experts." in name and not name.endswith(".weight"):
            name += ".weight"

        return super().filter_tensors((name, gen))
@@ -9764,73 +9709,6 @@ class MimoV2Model(TextModel):
                raise ValueError(f"Unprocessed experts: {experts}")


-@ModelBase.register("MiMoV2ForCausalLM")
-class MiMoV2VisionModel(MmprojModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        hp = self.hparams_vision
-
-        hp["image_size"] = hp.get("image_size", 560)
-        hp["num_attention_heads"] = hp.get("num_heads", 32)
-        hp["num_hidden_layers"] = hp.get("depth", 28)
-
-        self.n_q_heads = int(hp["num_heads"])
-        self.num_kv_heads = int(hp.get("num_key_value_heads", 8))
-        self.head_dim = int(hp.get("qk_channels", 64))
-        self.spatial_merge_size = int(hp["spatial_merge_size"])
-        # MiMoV2 vision RMSNorm: HF uses getattr(config, "rms_norm_eps", 1e-6) and the
-        # field is absent from MiMo-V2.5's vision_config
-        self.rms_norm_eps = float(hp.get("rms_norm_eps", 1e-6))
-
-        # fullatt_block_indexes are also reflected in vit_window_attn_types as -1
-        self.fullatt_block_indexes = list(hp.get("fullatt_block_indexes") or [])
-        self.vit_window_attn_types = list(hp.get("vit_window_attn_types") or [])
-        self.visual_token_window_size = int(hp.get("visual_token_window_size", -1))
-        self.use_sink = bool(hp.get("use_sink", False))
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MIMOVL)
-        self.gguf_writer.add_vision_use_silu(True)
-        self.gguf_writer.add_vision_head_count_kv(self.num_kv_heads)
-        self.gguf_writer.add_vision_spatial_merge_size(self.spatial_merge_size)
-        self.gguf_writer.add_uint32(gguf.Keys.ClipVision.WINDOW_SIZE, self.visual_token_window_size)
-        self.gguf_writer.add_vision_wa_pattern_mode(self.vit_window_attn_types)
-        self.gguf_writer.add_vision_attention_layernorm_eps(self.rms_norm_eps)
-        self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"]))
-        self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"]))
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        # Sinks must be F32: any sink-style softmax/mask add in ggml requires
-        # F32, and we fold sinks into a host-built F32 mask at encode time.
-        if new_name.endswith(".attn_sinks"):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, _ = item
-        if not name.startswith("visual."):
-            return None
-        return super().filter_tensors(item)
-
-    def modify_tensors(self, data_torch, name, bid):
-        # Conv3D patch embed: split along the temporal axis (kt=2) into two Conv2D
-        # weights that the existing qwen2vl-style two-Conv2D path consumes.
-        if name == "visual.patch_embed.proj.weight":
-            _, _, kt, _, _ = data_torch.shape
-            if kt != 2:
-                raise ValueError(f"unexpected temporal_patch_size: {kt}")
-            embd_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH]
-            yield (embd_name + ".weight",   data_torch[:, :, 0, ...])
-            yield (embd_name + ".weight.1", data_torch[:, :, 1, ...])
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
@ModelBase.register("Step3p5ForCausalLM")
 class Step35Model(TextModel):
    model_arch = gguf.MODEL_ARCH.STEP35
@@ -11689,34 +11567,6 @@ class BailingMoeV2Model(TextModel):
                raise ValueError(f"Unprocessed experts: {experts}")


-@ModelBase.register("SarvamMoEForCausalLM", "modeling_sarvam_moe.SarvamMoEForCausalLM")
-class SarvamMoEModel(BailingMoeV2Model):
-    model_arch = gguf.MODEL_ARCH.BAILINGMOE2
-    # Sarvam-MoE shares the BailingMoeV2 architecture; only differences:
-    #  - full rotary (no partial_rotary_factor)
-    #  - expert bias is zero-mean normalized at load time
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        hparams = self.hparams
-        if (rope_dim := hparams.get("head_dim")) is None:
-            rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
-        # Override the partial-rotary value written by BailingMoeV2 with the full rotary dim
-        self.gguf_writer.add_rope_dimension_count(rope_dim)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if name.endswith(".expert_bias"):
-            # Sarvam normalizes expert bias to zero mean
-            inner = gen
-
-            def gen():
-                t = inner()
-                return t - t.mean()
-        return super().filter_tensors((name, gen))
-
-
@ModelBase.register("GroveMoeForCausalLM", "modeling_grove_moe.GroveMoeForCausalLM")
 class GroveMoeModel(TextModel):
    model_arch = gguf.MODEL_ARCH.GROVEMOE
@@ -13413,7 +13263,7 @@ class PixtralModel(LlavaVisionModel):
        self.gguf_writer.add_vision_use_silu(True)

        # spatial_merge_size
-        if self.find_vparam(["mm_projector_id"], optional=True) == "patch_merge":
+        if self.find_vparam(["mm_projector_id"]) == "patch_merge":
            self.gguf_writer.add_vision_spatial_merge_size(
                self.find_vparam(["spatial_merge_size"])
            )
@@ -13421,12 +13271,8 @@ class PixtralModel(LlavaVisionModel):
    def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:
        if name == "vision_language_adapter.w_in.weight":
            return "mm.1.weight"
-        elif name == "vision_language_adapter.w_in.bias":
-            return "mm.1.bias"
        elif name == "vision_language_adapter.w_out.weight":
            return "mm.2.weight"
-        elif name == "vision_language_adapter.w_out.bias":
-            return "mm.2.bias"
        return super().map_tensor_name(name, try_suffixes)


@@ -13838,27 +13684,6 @@ class DotsOCRVisionModel(MmprojModel):
        yield from super().modify_tensors(data_torch, name, bid)


-@ModelBase.register("Sarashina2VisionForCausalLM")
-class Sarashina2VLTextModel(LlamaModel):
-    model_arch = gguf.MODEL_ARCH.LLAMA
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if name.startswith("llm."):
-            name = name.replace("llm.", "", 1)
-        elif name.startswith("norm."):
-            return None
-        return super().filter_tensors((name, gen))
-
-
-@ModelBase.register("Sarashina2VisionForCausalLM")
-class Sarashina2VLVisionModel(Qwen2VLVisionModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.global_config['model_type'] = "qwen2_vl"
-
-
 ###### CONVERSION LOGIC ######


@@ -14115,7 +13940,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
    # Step3-VL keeps text config under text_config but uses a custom top-level architecture.
    # For text conversion we route to a dedicated text-only class.
    # TODO: refactor this later to avoid adding exception here
-    if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM"):
+    if model_type == ModelType.TEXT and arch == "StepVLForConditionalGeneration":
        return arch

    # if "architectures" is found in the sub-config, use that instead
@@ -155,7 +155,6 @@ models = [
    {"name": "joyai-llm",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jdopensource/JoyAI-LLM-Flash", },
    {"name": "kanana2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/kakaocorp/kanana-2-30b-a3b-instruct-2601", },
    {"name": "f2llmv2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/codefuse-ai/F2LLM-v2-4B", },
-    {"name": "sarvam-moe",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sarvamai/sarvam-30b", },
 ]

 # some models are known to be broken upstream, so we will skip them as exceptions
@@ -188,24 +188,6 @@ class LoraTorchTensor:
    def swapaxes(self, axis0: int, axis1: int) -> LoraTorchTensor:
        return self.transpose(axis0, axis1)

-    def split(self, split_size: int | Sequence[int], dim: int = 0) -> tuple[LoraTorchTensor, ...]:
-        shape = self.shape
-        ndim = len(shape)
-        if dim < 0:
-            dim += ndim
-        if dim == ndim - 1:
-            A_chunks = self._lora_A.split(split_size, dim=-1)
-            return tuple(LoraTorchTensor(a, self._lora_B) for a in A_chunks)
-        elif dim == ndim - 2:
-            B_chunks = self._lora_B.split(split_size, dim=-2)
-            return tuple(LoraTorchTensor(self._lora_A, b) for b in B_chunks)
-        else:
-            B_chunks = self._lora_B.split(split_size, dim=dim)
-            if self._lora_A.shape[dim] == 1:
-                return tuple(LoraTorchTensor(self._lora_A, b) for b in B_chunks)
-            A_chunks = self._lora_A.split(split_size, dim=dim)
-            return tuple(LoraTorchTensor(a, b) for a, b in zip(A_chunks, B_chunks))
-
    def to(self, *args, **kwargs):
        return LoraTorchTensor(self._lora_A.to(*args, **kwargs), self._lora_B.to(*args, **kwargs))

@@ -248,11 +230,6 @@ class LoraTorchTensor:
                )
            else:
                raise NotImplementedError
-        elif func is torch.split:
-            assert len(args) and len(args) >= 2
-            tensor, split_size = args[0], args[1]
-            dim = args[2] if len(args) > 2 else kwargs.get("dim", 0)
-            return tensor.split(split_size, dim=dim)
        else:
            raise NotImplementedError

@@ -57,22 +57,17 @@ Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvin

 ## Validated Models

-The following models were validated on Intel® Core™ Ultra Series 2. While our testing was limited, the OpenVINO backend is expected to work across a broad range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html).
- Use `GGML_OPENVINO_STATEFUL_EXECUTION=1` when using GPU device.
- `-fa 1` is required when running llama-bench with the OpenVINO backend.
- Additional model support, quantization formats and validations are work in progress.
+The following models have been validated for functionality on Intel® Core™ Ultra Series 1 and Series 2:

-| Model  | Validated   | Known Issues  |
-| :------| :---------- | :-------------|
-| [Llama-3.2-1B-Instruct](https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — |
-| [Meta-Llama-3.1-8B-Instruct](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | `Q8_0`, `Q4_K_M` on CPU/GPU/NPU | `Q4_0_8_8`, `Q4_0_4_8`, `Q4_0_4_4` fail |
-| [Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf) | `FP16`, `Q4` on CPU/NPU | GPU unsupported for `FP16` and `Q4` (`llama-cli`, `llama-bench`) |
-| [Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — |
-| [Qwen3-8B-Instruct](https://huggingface.co/Qwen/Qwen3-8B-GGUF) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/NPU; GPU works via `llama-bench` | GPU `llama-cli` unsupported for all quantizations |
-| [MiniCPM-V-2_6-GGUF](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `Q4_0` on CPU/GPU/NPU | — |
-| [DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — |
-| [Hunyuan-7B-Instruct](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | CPU: `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M`; GPU: `Q8_0`, `Q4_0`, `Q4_1`; NPU (`llama-bench` only): `Q4_0`, `Q4_1`, `Q4_K_M` | GPU `Q4_K_M` unsupported; NPU `llama-cli` unsupported |
-| [Mistral-7B-Instruct-v0.3](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF/) | CPU/GPU: `Q8_0`, `Q4_K_M`; NPU: `Q8_0`, `Q4_K_M` (via `llama-bench`) | NPU `llama-cli` unsupported for `Q8_0`, `Q4_K_M` |
+- [Llama-3.2-1B-Instruct-GGUF](https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/)
+- [Llama-3.1-8B-Instruct](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF)
+- [microsoft/Phi-3-mini-4k-instruct-gguf](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf)
+- [Qwen/Qwen2.5-1.5B-Instruct-GGUF](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF)
+- [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B-GGUF)
+- [openbmb/MiniCPM-1B-sft-bf16](https://huggingface.co/openbmb/MiniCPM-S-1B-sft-gguf)
+- [tencent/Hunyuan-7B-Instruct](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF)
+- [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF)
+- [bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF)

 ## Build Instructions

@@ -720,7 +720,6 @@ use 1 SYCL GPUs: [0] with Max compute units:512
 | GGML_SYCL_GRAPH    | OFF *(default)* \|ON *(Optional)*     | Enable build with [SYCL Graph extension](https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc). |
 | GGML_SYCL_DNN      | ON *(default)* \|OFF *(Optional)*     | Enable build with oneDNN.                   |
 | GGML_SYCL_HOST_MEM_FALLBACK | ON *(default)* \|OFF *(Optional)* | Allow host memory fallback when device memory is full during quantized weight reorder. Enables inference to continue at reduced speed (reading over PCIe) instead of failing. Requires Linux kernel 6.8+. |
-| GGML_SYCL_SUPPORT_LEVEL_ZERO | ON *(default)* \|OFF *(Optional)* | Enable Level Zero API for device memory allocation. Requires Level Zero headers/library at build time and Intel GPU driver (Level Zero runtime) at run time. Reduces system RAM usage during multi-GPU inference. |
 | CMAKE_C_COMPILER   | `icx` *(Linux)*, `icx/cl` *(Windows)* | Set `icx` compiler for SYCL code path.      |
 | CMAKE_CXX_COMPILER | `icpx` *(Linux)*, `icx` *(Windows)*   | Set `icpx/icx` compiler for SYCL code path. |

@@ -734,18 +733,9 @@ use 1 SYCL GPUs: [0] with Max compute units:512
 | GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.|
 | GGML_SYCL_DISABLE_OPT | 0 (default) or 1 | Disable optimize features for Intel GPUs. (Recommended to 1 for intel devices older than Gen 10) |
 | GGML_SYCL_DISABLE_GRAPH | 0 or 1 (default) | Disable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. |
-| GGML_SYCL_ENABLE_LEVEL_ZERO | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO=ON at build time. |
 | GGML_SYCL_DISABLE_DNN | 0 (default) or 1 | Disable running computations through oneDNN and always use oneMKL. |
 | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.<br>Recommended to use when --split-mode = layer |
-| UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. |
-
-## Compile-time Flags
-
-Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spot.
-
-| Name            | Function                                                                         |
-|-----------------|----------------------------------------------------------------------------------|
-| DEBUG_SYCL_POOL | Enable device memory pool logging on teardown. Useful for profiling allocations. |
+| UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Support malloc device memory more than 4GB.|

 ## Design Rule

@@ -821,7 +811,7 @@ Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spo

 - `ggml_backend_sycl_buffer_type_alloc_buffer: can't allocate 5000000000 Bytes of memory on device`

-  With the default `GGML_SYCL_ENABLE_LEVEL_ZERO=1`, llama.cpp requests Level Zero's relaxed maximum-size allocation limit directly. If Level Zero support is disabled at build time or runtime and the allocation goes through SYCL/Unified Runtime instead, enable support for allocations larger than 4 GiB by:
+  You need to enable to support 4GB memory malloc by:
  ```
    export UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1
    set UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1
@@ -9,20 +9,18 @@ wget https://archive.spacemit.com/toolchain/spacemit-toolchain-linux-glibc-x86_6
 ~~~

 2. Build
-Below is the build script: it requires utilizing RISC-V vector instructions for acceleration. Ensure the `GGML_CPU_RISCV64_SPACEMIT` compilation option is enabled. The currently supported optimization version is `RISCV64_SPACEMIT_IME1` and `RISCV64_SPACEMIT_IME2`, corresponding to the `RISCV64_SPACEMIT_IME_SPEC` compilation option. Compiler configurations are defined in the `riscv64-spacemit-linux-gnu-gcc.cmake` file. Please ensure you have installed the RISC-V compiler and set the environment variable via `export RISCV_ROOT_PATH={your_compiler_path}`.
+Below is the build script: it requires utilizing RISC-V vector instructions for acceleration. Ensure the `GGML_CPU_RISCV64_SPACEMIT` compilation option is enabled. The currently supported optimization version is `RISCV64_SPACEMIT_IME1`, corresponding to the `RISCV64_SPACEMIT_IME_SPEC` compilation option. Compiler configurations are defined in the `riscv64-spacemit-linux-gnu-gcc.cmake` file. Please ensure you have installed the RISC-V compiler and set the environment variable via `export RISCV_ROOT_PATH={your_compiler_path}`.
 ```bash

 cmake -B build \
    -DCMAKE_BUILD_TYPE=Release \
    -DGGML_CPU_RISCV64_SPACEMIT=ON \
-    -DGGML_CPU_REPACK=OFF \
    -DLLAMA_OPENSSL=OFF \
    -DGGML_RVV=ON \
-    -DGGML_RV_ZVFH=ON \
    -DGGML_RV_ZFH=ON \
    -DGGML_RV_ZICBOP=ON \
    -DGGML_RV_ZIHINTPAUSE=ON \
-    -DGGML_RV_ZBA=ON \
+    -DRISCV64_SPACEMIT_IME_SPEC=RISCV64_SPACEMIT_IME1 \
    -DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake \
    -DCMAKE_INSTALL_PREFIX=build/installed

@@ -49,25 +47,8 @@ export RISCV_ROOT_PATH_IME1={your RISC-V compiler path}

 ${QEMU_ROOT_PATH}/bin/qemu-riscv64 -L ${RISCV_ROOT_PATH_IME1}/sysroot -cpu max,vlen=256,elen=64,vext_spec=v1.0 ${PWD}/build/bin/llama-cli -m ${PWD}/models/Qwen2.5-0.5B-Instruct-Q4_0.gguf -t 1
 ~~~
-
-## Quantization Support For Matrix
-
-| Quantization Type | X60 | A100 |
-| ---: | ---: | ---: |
-| Q2_K |  | :heavy_check_mark: |
-| Q3_K |  | :heavy_check_mark: |
-| Q4_0 | :heavy_check_mark: | :heavy_check_mark: |
-| Q4_1 | :heavy_check_mark: | :heavy_check_mark: |
-| Q4_K | :heavy_check_mark: | :heavy_check_mark: |
-| Q5_0 |  | :heavy_check_mark: |
-| Q5_1 |  | :heavy_check_mark: |
-| Q5_K |  | :heavy_check_mark: |
-| Q6_K |  | :heavy_check_mark: |
-| Q8_0 |  | :heavy_check_mark: |
-
-
 ## Performance
-* Spacemit(R) X60
+#### Quantization Support For Matrix
 ~~~
 model name      : Spacemit(R) X60
 isa             : rv64imafdcv_zicbom_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zfhmin_zca_zcd_zba_zbb_zbc_zbs_zkt_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt
@@ -77,34 +58,33 @@ mvendorid       : 0x710
 marchid         : 0x8000000058000001
 ~~~

-| model                          |       size |     params | backend    | threads | n_ubatch | fa | mmap |            test |                  t/s |
-| ------------------------------ | ---------: | ---------: | ---------- | ------: | -------: | -: | ---: | --------------: | -------------------: |
-| qwen35 2B Q4_1                 |   1.19 GiB |     1.88 B | CPU        |       4 |      128 |  1 |    0 |           pp128 |         10.32 ± 0.02 |
-| qwen35 2B Q4_1                 |   1.19 GiB |     1.88 B | CPU        |       4 |      128 |  1 |    0 |           tg128 |          3.07 ± 0.01 |
-| qwen3 0.6B Q4_0                | 358.78 MiB |   596.05 M | CPU        |       4 |      128 |  1 |    0 |           pp128 |         49.15 ± 0.25 |
-| qwen3 0.6B Q4_0                | 358.78 MiB |   596.05 M | CPU        |       4 |      128 |  1 |    0 |           tg128 |         11.73 ± 0.02 |
+Q4_0
+|   Model    |   Size   | Params | backend | threads | test | t/s |
+| -----------| -------- | ------ | ------- | ------- | ---- |------|
+Qwen2.5 0.5B |403.20 MiB|630.17 M|   cpu   |    4    | pp512|64.12 ± 0.26|
+Qwen2.5 0.5B |403.20 MiB|630.17 M|   cpu   |    4    | tg128|10.03 ± 0.01|
+Qwen2.5 1.5B |1011.16 MiB| 1.78 B |   cpu   |    4    | pp512|24.16 ± 0.02|
+Qwen2.5 1.5B |1011.16 MiB| 1.78 B |   cpu   |    4    | tg128|3.83 ± 0.06|
+Qwen2.5 3B   | 1.86 GiB  | 3.40 B |   cpu   |    4    | pp512|12.08 ± 0.02|
+Qwen2.5 3B   | 1.86 GiB  | 3.40 B |   cpu   |    4    | tg128|2.23 ± 0.02|
+
+Q4_1
+|   Model    |   Size   | Params | backend | threads | test | t/s |
+| -----------| -------- | ------ | ------- | ------- | ---- |------|
+Qwen2.5 0.5B |351.50 MiB|494.03 M|   cpu   |    4    | pp512|62.07 ± 0.12|
+Qwen2.5 0.5B |351.50 MiB|494.03 M|   cpu   |    4    | tg128|9.91 ± 0.01|
+Qwen2.5 1.5B |964.06 MiB| 1.54 B |   cpu   |    4    | pp512|22.95 ± 0.25|
+Qwen2.5 1.5B |964.06 MiB| 1.54 B |   cpu   |    4    | tg128|4.01 ± 0.15|
+Qwen2.5 3B   | 1.85 GiB | 3.09 B |   cpu   |    4    | pp512|11.55 ± 0.16|
+Qwen2.5 3B   | 1.85 GiB | 3.09 B |   cpu   |    4    | tg128|2.25 ± 0.04|


-* Spacemit(R) A100
-~~~
-model name      : Spacemit(R) A100
-isa             : rv64imafdcvh_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintntl_zihintpause_zihpm_zimop_zaamo_zalrsc_zawrs_zfa_zfh_zfhmin_zca_zcb_zcd_zcmop_zba_zbb_zbc_zbs_zkt_zvbb_zvbc_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkb_zvkg_zvkned_zvknha_zvknhb_zvksed_zvksh_zvkt_smaia_smstateen_ssaia_sscofpmf_sstc_svinval_svnapot_svpbmt_sdtrig
-mmu             : sv39
-mvendorid       : 0x710
-marchid         : 0x8000000041000002
-mimpid          : 0x10000000d5686200
-hart isa        : rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintntl_zihintpause_zihpm_zimop_zaamo_zalrsc_zawrs_zfa_zfh_zfhmin_zca_zcb_zcd_zcmop_zba_zbb_zbc_zbs_zkt_zvbb_zvbc_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkb_zvkg_zvkned_zvknha_zvknhb_zvksed_zvksh_zvkt_smaia_smstateen_ssaia_sscofpmf_sstc_svinval_svnapot_svpbmt_sdtrig
-~~~
-
-| model                          |       size |     params | backend    | threads | n_ubatch | fa | mmap |            test |                  t/s |
-| ------------------------------ | ---------: | ---------: | ---------- | ------: | -------: | -: | ---: | --------------: | -------------------: |
-| qwen3 0.6B Q4_0                | 358.78 MiB |   596.05 M | CPU        |       8 |      128 |  1 |    0 |           pp128 |        565.83 ± 0.31 |
-| qwen3 0.6B Q4_0                | 358.78 MiB |   596.05 M | CPU        |       8 |      128 |  1 |    0 |           tg128 |         55.77 ± 0.02 |
-| qwen3 4B Q4_0                  |   2.21 GiB |     4.02 B | CPU        |       8 |      128 |  1 |    0 |           pp128 |         79.74 ± 0.04 |
-| qwen3 4B Q4_0                  |   2.21 GiB |     4.02 B | CPU        |       8 |      128 |  1 |    0 |           tg128 |         11.29 ± 0.00 |
-| qwen3moe 30B.A3B Q4_0          |  16.18 GiB |    30.53 B | CPU        |       8 |      128 |  1 |    0 |           pp128 |         57.88 ± 0.31 |
-| qwen3moe 30B.A3B Q4_0          |  16.18 GiB |    30.53 B | CPU        |       8 |      128 |  1 |    0 |           tg128 |         12.79 ± 0.00 |
-| qwen35 2B Q4_1                 |   1.19 GiB |     1.88 B | CPU        |       8 |      128 |  1 |    0 |           pp128 |        115.23 ± 0.04 |
-| qwen35 2B Q4_1                 |   1.19 GiB |     1.88 B | CPU        |       8 |      128 |  1 |    0 |           tg128 |         16.49 ± 0.01 |
-| gemma4 E4B Q4_K - Medium       |   4.76 GiB |     7.52 B | CPU        |       8 |      128 |  1 |    0 |           pp128 |         21.13 ± 0.01 |
-| gemma4 E4B Q4_K - Medium       |   4.76 GiB |     7.52 B | CPU        |       8 |      128 |  1 |    0 |           tg128 |          5.66 ± 0.00 |
+Q4_K
+|   Model    |   Size   | Params | backend | threads | test | t/s |
+| -----------| -------- | ------ | ------- | ------- | ---- |------|
+Qwen2.5 0.5B |462.96 MiB|630.17 M|   cpu   |    4    | pp512|9.29 ± 0.05|
+Qwen2.5 0.5B |462.96 MiB|630.17 M|   cpu   |    4    | tg128|5.67 ± 0.04|
+Qwen2.5 1.5B | 1.04 GiB | 1.78 B |   cpu   |    4    | pp512|10.38 ± 0.10|
+Qwen2.5 1.5B | 1.04 GiB | 1.78 B |   cpu   |    4    | tg128|3.17 ± 0.08|
+Qwen2.5 3B   | 1.95 GiB | 3.40 B |   cpu   |    4    | pp512|4.23 ± 0.04|
+Qwen2.5 3B   | 1.95 GiB | 3.40 B |   cpu   |    4    | tg128|1.73 ± 0.00|
@@ -18,7 +18,7 @@ Legend:
 |                              ACC | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
 |                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             ADD1 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
-|                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           ARANGE | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          ARGSORT | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
@@ -61,17 +61,16 @@ Legend:
 |                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                        HARDSWISH | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                           IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
 |                          L2_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                       LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ❌ | ❌ | ❌ |
 |                              LOG | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
 |                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
-|                 MUL_MAT_HADAMARD | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
 |                              NEG | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
-|                             NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
+|                             NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ | ❌ |
 |                   OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
 |                     OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
 |                         OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | 🟡 |
@@ -118,5 +117,5 @@ Legend:
 |                            TOP_K | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
 |                              TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                            TRUNC | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
-|                          UPSCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                          UPSCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                            XIELU | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ |
@@ -1,26 +0,0 @@
-# llama-eval
-
-Simple evaluation tool for llama.cpp with support for multiple datasets.
-
-For a full description, usage examples, and sample results, see:
-
- [PR 21152](https://github.com/ggml-org/llama.cpp/pull/21152)
-
-## Quick start
-
-```bash
-# Single server
-python3 llama-eval.py \
-  --server http://localhost:8033 \
-  --model my-model \
-  --dataset gsm8k --n_cases 100 \
-  --grader-type regex --threads 32
-
-# Multiple servers (comma-separated URLs and thread counts)
-python3 llama-eval.py \
-  --server http://server1:8033,http://server2:8033 \
-  --server-name server1,server2 \
-  --threads 16,16 \
-  --dataset aime2025 --n_cases 240 \
-  --grader-type regex
-```
@@ -1,317 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import json
-import random
-import re
-import time
-import sys
-import os
-import threading
-from http.server import HTTPServer, BaseHTTPRequestHandler
-from typing import Dict, List, Optional
-from dataclasses import dataclass
-from pathlib import Path
-
-import datasets
-
-# Set cache directory for HuggingFace datasets
-cache_dir = Path.home() / ".cache" / "huggingface" / "datasets"
-cache_dir.mkdir(parents=True, exist_ok=True)
-os.environ["HF_DATASETS_CACHE"] = str(cache_dir)
-
-def dice(s1: str, s2: str) -> float:
-    """Calculate Dice coefficient between two strings based on bigram overlap."""
-    if not s1 and not s2:
-        return 1.0
-
-    def _bigrams(s: str):
-        return [s[i : i + 2] for i in range(len(s) - 1)]
-
-    bigrams1 = _bigrams(s1)
-    bigrams2 = _bigrams(s2)
-
-    if not bigrams1 and not bigrams2:
-        return 1.0
-
-    from collections import Counter
-
-    freq1 = Counter(bigrams1)
-    freq2 = Counter(bigrams2)
-
-    intersection = sum(min(freq1[bg], freq2[bg]) for bg in freq1)
-    dice_coeff = 2 * intersection / (len(bigrams1) + len(bigrams2))
-    return dice_coeff
-
-def debug_log(message: str):
-    """Log debug messages to both stdout and a file"""
-    print(message, file=sys.stderr)
-    with open("/tmp/simulator-debug.log", "a") as f:
-        f.write(message + "\n")
-
-simulator: Optional["Simulator"] = None
-
-@dataclass
-class EvalState:
-    id: str
-    tasks: List[str]
-    task_states: Dict[str, Dict]
-    sampling_config: Dict
-
-def normalize_number(s: str) -> Optional[int]:
-    match = re.match(r"\d+", s)  # match digits from the start
-    if not match:
-        return None
-    return int(match.group(0))
-
-class AimeDataset:
-    def __init__(self, split: str = "train"):
-        self.split = split
-        self.questions: List[Dict] = []
-        self._load_dataset()
-
-    def _load_dataset(self):
-        print(f"Loading AIME dataset (split: {self.split})...")
-
-        cache_path = Path.home() / ".cache" / "huggingface" / "datasets" / "AI-MO___aimo-validation-aime" / "default" / "0.0.0"
-        if cache_path.exists():
-            print(f"Using cached dataset from {cache_path}")
-            ds = datasets.load_dataset("AI-MO/aimo-validation-aime", split=self.split, cache_dir=str(cache_path))
-        else:
-            ds = datasets.load_dataset("AI-MO/aimo-validation-aime", split=self.split)
-
-        self.questions = list(ds)
-        print(f"AIME dataset loaded: {len(self.questions)} questions")
-
-    def find_question(self, request_text: str) -> Optional[Dict]:
-        best_match = None
-        best_distance = -1
-        best_index = -1
-
-        for i, question in enumerate(self.questions):
-            question_text = question["problem"]
-            request_lower = request_text.lower()
-            question_lower = question_text.lower()
-
-            # Exact match
-            if question_lower == request_lower:
-                debug_log(f"DEBUG: Found exact match at index {i}")
-                return question
-
-            # Remove LaTeX formatting for more flexible matching
-            question_no_latex = re.sub(r'\$[^$]+\$', '', question_text)
-            if question_no_latex.lower() == request_lower:
-                debug_log(f"DEBUG: Found match (no LaTeX) at index {i}")
-                return question
-
-            # Calculate Dice coefficient for partial matches
-            # Only consider if request is at least 50% of question length
-            if len(request_lower) >= len(question_lower) * 0.5:
-                distance = dice(question_lower, request_lower)
-
-                if distance > best_distance:
-                    best_distance = distance
-                    best_match = question
-                    best_index = i
-
-        if best_match and best_distance > 0.3:  # Threshold for partial match
-            debug_log(f"DEBUG: Found best partial match at index {best_index} with distance {best_distance:.3f}")
-            return best_match
-
-        debug_log(f"DEBUG: No matching question found for: {request_text[:100]}...")
-        return None
-
-    def get_answer(self, question: Dict) -> str:
-        answer = question["answer"]
-        if isinstance(answer, str):
-            normalized = normalize_number(answer)
-            return str(normalized) if normalized is not None else answer
-        return str(answer)
-
-class Simulator:
-    def __init__(
-        self,
-        port: int = 8033,
-        host: str = "localhost",
-        success_rate: float = 0.8,
-        dataset_split: str = "train"
-    ):
-        self.port = port
-        self.host = host
-        self.success_rate = success_rate
-        self.dataset = AimeDataset(dataset_split)
-        self.eval_state = EvalState(
-            id="aime-2025",
-            tasks=["aime"],
-            task_states={},
-            sampling_config={"temperature": 0, "max_tokens": 2048}
-        )
-
-    def _generate_response(
-        self,
-        question: Dict,
-        should_be_correct: bool
-    ) -> Dict:
-        expected_answer = self.dataset.get_answer(question)
-
-        if should_be_correct:
-            response_text = expected_answer
-        else:
-            response_text = self._generate_wrong_answer(question)
-
-        return {
-            "id": f"chatcmpl-{int(time.time())}",
-            "object": "chat.completion",
-            "created": int(time.time()),
-            "model": "llama",
-            "choices": [
-                {
-                    "index": 0,
-                    "message": {
-                        "role": "assistant",
-                        "content": response_text
-                    },
-                    "finish_reason": "stop"
-                }
-            ],
-            "usage": {
-                "prompt_tokens": 100,
-                "completion_tokens": 50,
-                "total_tokens": 150
-            }
-        }
-
-    def _generate_wrong_answer(self, question: Dict) -> str:
-        expected_answer = self.dataset.get_answer(question)
-
-        if expected_answer.isdigit():
-            wrong_answer = str(int(expected_answer) + 1)
-        else:
-            wrong_answer = expected_answer + " (wrong)"
-
-        return wrong_answer
-
-    def _process_request(self, request_data: Dict) -> Dict:
-        messages = request_data.get("messages", [])
-        if not messages:
-            return {"error": "No messages in request"}
-
-        request_text = messages[0].get("content", "")
-        debug_log(f"DEBUG: Received request with content: {request_text[:150]}...")
-
-        question = self.dataset.find_question(request_text)
-        if not question:
-            debug_log(f"DEBUG: find_question returned None")
-            return {"error": "No matching question found"}
-
-        should_be_correct = random.random() < self.success_rate
-
-        response = self._generate_response(question, should_be_correct)
-
-        task_id = "aime"
-        self.eval_state.task_states[task_id] = {
-            "correct": should_be_correct,
-            "expected": self.dataset.get_answer(question),
-            "predicted": response["choices"][0]["message"]["content"]
-        }
-
-        return response
-
-class RequestHandler(BaseHTTPRequestHandler):
-    def do_POST(self):
-        if self.path != "/v1/chat/completions":
-            self._send_json({"error": "Not found"}, 404)
-            return
-
-        try:
-            content_length = int(self.headers.get("Content-Length", 0))
-            body = self.rfile.read(content_length)
-            request_data = json.loads(body) if body else None
-
-            if not request_data:
-                self._send_json({"error": "Invalid JSON"}, 400)
-                return
-
-            if simulator is None:
-                self._send_json({"error": "Simulator not initialized"}, 500)
-                return
-
-            response = simulator._process_request(request_data)
-            self._send_json(response, 200)
-
-        except json.JSONDecodeError:
-            self._send_json({"error": "Invalid JSON"}, 400)
-        except Exception as e:
-            print(f"Error processing request: {e}")
-            self._send_json({"error": str(e)}, 500)
-
-    def _send_json(self, data: dict, status: int = 200):
-        body = json.dumps(data).encode("utf-8")
-        self.send_response(status)
-        self.send_header("Content-Type", "application/json")
-        self.send_header("Content-Length", str(len(body)))
-        self.end_headers()
-        self.wfile.write(body)
-
-    def log_message(self, format, *args):
-        # Suppress default request logging
-        pass
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="llama-server simulator for testing eval scripts"
-    )
-    parser.add_argument(
-        "--port",
-        type=int,
-        default=8033,
-        help="Server port (default: 8033)"
-    )
-    parser.add_argument(
-        "--host",
-        type=str,
-        default="localhost",
-        help="Server host (default: localhost)"
-    )
-    parser.add_argument(
-        "--success-rate",
-        type=float,
-        default=0.8,
-        help="Success rate 0-1 (default: 0.8)"
-    )
-    parser.add_argument(
-        "--dataset-split",
-        type=str,
-        default="train",
-        help="AIME dataset split to use (default: train)"
-    )
-
-    args = parser.parse_args()
-
-    global simulator
-    simulator = Simulator(
-        port=args.port,
-        host=args.host,
-        success_rate=args.success_rate,
-        dataset_split=args.dataset_split
-    )
-
-    server = HTTPServer((args.host, args.port), RequestHandler)
-    server_thread = threading.Thread(target=server.serve_forever, daemon=True)
-    server_thread.start()
-
-    print("\n=== llama-server-simulator ===")
-    print(f"Server running on http://{args.host}:{args.port}")
-    print(f"Success rate: {args.success_rate}")
-    print(f"AIME dataset loaded: {len(simulator.dataset.questions)} questions")
-    print("\nPress Ctrl+C to stop\n")
-
-    try:
-        server_thread.join()
-    except KeyboardInterrupt:
-        print("\nShutting down...")
-        server.shutdown()
-
-if __name__ == "__main__":
-    main()
@@ -1,86 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Get the directory where this script is located
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-echo "=== llama-server-simulator Test Script ==="
-echo ""
-
-PORT=8033
-SUCCESS_RATE=0.8
-TEST_PORT=8034
-
-echo "Starting simulator on port $PORT with success rate $SUCCESS_RATE..."
-source "$SCRIPT_DIR/venv/bin/activate"
-python3 "$SCRIPT_DIR/llama-server-simulator.py" --port $PORT --success-rate $SUCCESS_RATE > /tmp/simulator-test.log 2>&1 &
-SIMULATOR_PID=$!
-
-echo "Waiting for simulator to start..."
-sleep 5
-
-# Helper function to make a request and extract the answer
-make_request() {
-  local question="$1"
-  curl -s -X POST http://localhost:$PORT/v1/chat/completions \
-    -H "Content-Type: application/json" \
-    -d "{
-      \"model\": \"llama\",
-      \"messages\": [
-        {\"role\": \"user\", \"content\": \"$question\"}
-      ],
-      \"temperature\": 0,
-      \"max_tokens\": 2048
-    }" | python3 -c "import sys, json; data = json.load(sys.stdin); print(data.get('choices', [{}])[0].get('message', {}).get('content', data.get('error', 'No response')))"
-}
-
-# Test question (repeated in multiple tests)
-TEST_QUESTION="Quadratic polynomials P(x) and Q(x) have leading coefficients 2 and -2, respectively. The graphs of both polynomials pass through the two points (16,54) and (20,53). Find P(0) + Q(0)."
-
-echo ""
-echo "=== Test 1: Correct Answer ==="
-echo "Sending request with known question..."
-answer=$(make_request "$TEST_QUESTION")
-echo "Answer: $answer"
-echo "Expected: 116"
-echo "Correct: $([ "$answer" == "116" ] && echo "Yes" || echo "No")"
-
-echo ""
-echo "=== Test 2: Wrong Answer ==="
-echo "Sending request with known question (success rate 0.0)..."
-answer=$(make_request "$TEST_QUESTION")
-echo "Answer: $answer"
-echo "Expected: 116"
-echo "Correct: $([ "$answer" == "116" ] && echo "Yes" || echo "No")"
-
-echo ""
-echo "=== Test 3: No Matching Question ==="
-echo "Sending request with non-matching text..."
-response=$(make_request "What is the capital of France?")
-echo "Response: $response"
-echo "Expected: No matching question found"
-echo "Correct: $([ "$response" == "No matching question found" ] && echo "Yes" || echo "No")"
-
-echo ""
-echo "=== Test 4: Success Rate Verification ==="
-echo "Sending 10 requests to test success rate..."
-correct_count=0
-for i in {1..10}; do
-  answer=$(make_request "$TEST_QUESTION")
-  if [ "$answer" == "116" ]; then
-    correct_count=$((correct_count + 1))
-  fi
-  echo "  Request $i: Answer = $answer"
-done
-echo "Correct answers: $correct_count/10"
-echo "Expected: ~8/10 (80% success rate)"
-echo "Success rate: $(echo "scale=1; $correct_count * 10" | bc)%"
-
-echo ""
-echo "=== Test Complete ==="
-echo "Stopping simulator..."
-kill $SIMULATOR_PID 2>/dev/null
-wait $SIMULATOR_PID 2>/dev/null || true
-
-echo "Simulator stopped."
@@ -52,10 +52,6 @@ causal-convert-mm-model:
 	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
 	./scripts/causal/convert-model.sh

-	$(MAKE) causal-convert-mmproj MM_OUTTYPE="$(MM_OUTTYPE)"
-
-causal-convert-mmproj:
-	$(call validate_model_path,causal-convert-mmproj)
 	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(MM_OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \
 	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
 	./scripts/causal/convert-model.sh --mmproj
@@ -6,7 +6,7 @@ Demonstration of basic greedy speculative decoding
 ./bin/llama-speculative-simple \
    -m  ../models/qwen2.5-32b-coder-instruct/ggml-model-q8_0.gguf \
    -md ../models/qwen2.5-1.5b-coder-instruct/ggml-model-q4_0.gguf \
-    -f test.txt -c 0 -ngl 99 --color on \
-    --sampling-seq k --top-k 1 -fa on --temp 0.0 \
-    -ngld 99 --spec-draft-n-max 16 --spec-draft-n-draft-min 5 --draft-p-min 0.9
+    -f test.txt -c 0 -ngl 99 --color \
+    --sampling-seq k --top-k 1 -fa --temp 0.0 \
+    -ngld 99 --draft-max 16 --draft-min 5 --draft-p-min 0.9
 ```
@@ -145,9 +145,9 @@ int main(int argc, char ** argv) {
    // init the speculator
    const auto & params_spec = params.speculative;

-    struct common_speculative * spec = common_speculative_init(params.speculative, 1);
+    struct common_speculative * spec = common_speculative_init(params.speculative, seq_id);

-    common_speculative_begin(spec, seq_id, prompt_tgt);
+    common_speculative_begin(spec, prompt_tgt);

    llama_batch batch_tgt = llama_batch_init(llama_n_batch(ctx_tgt), 0, 1);

@@ -179,15 +179,7 @@ int main(int argc, char ** argv) {
            }

            // generate a new draft
-            common_speculative_get_draft_params(spec, seq_id) = {
-                /* .drafting   = */ true,
-                /* .n_max      = */ -1,
-                /* .n_past     = */ n_past,
-                /* .id_last    = */ id_last,
-                /* .prompt     = */ &prompt_tgt,
-                /* .result     = */ &draft, // output
-            };
-            common_speculative_draft(spec);
+            draft = common_speculative_draft(spec, params_spec, prompt_tgt, prompt_tgt.size(), id_last);

            // save the original draft size
            n_draft = draft.size();
@@ -280,7 +272,7 @@ int main(int argc, char ** argv) {
            continue;
        }

-        common_speculative_accept(spec, seq_id, ids.size() - 1);
+        common_speculative_accept(spec, ids.size() - 1);

        // full acceptance: consume the draft and commit accepted tokens
        n_past    += ids.size() - 1;
@@ -111,14 +111,14 @@ if [ $GGML_SYCL_DEVICE -ne -1 ]; then
    echo "Use $GGML_SYCL_DEVICE as main GPU"
    #use signle GPU only
    GPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}"
-    export ONEAPI_DEVICE_SELECTOR="level_zero:${GGML_SYCL_DEVICE}"
+    export ONEAPI_DEVICE_SELECTOR="level_zero:${$GGML_SYCL_DEVICE}"
    echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"
 else
    echo "Use all Intel GPUs, including iGPU & dGPU"
    GPUS_SETTING="-sm ${SPLIT_MODE}"
 fi

-echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE}  --mmap --host 0.0.0.0 --port 8000"
+echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE}  --mmap "
 ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap --host 0.0.0.0 --port 8000


@@ -0,0 +1,58 @@
+{
+  "nodes": {
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1730504689,
+        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1732014248,
+        "narHash": "sha256-y/MEyuJ5oBWrWAic/14LaIr/u5E0wRVzyYsouYY3W6w=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "23e89b7da85c3640bbc2173fe04f4bd114342367",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1730504152,
+        "narHash": "sha256-lXvH/vOfb4aGYyvFmZK/HlsNsr/0CVWlwYvo2rxJk3s=",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
@@ -5,7 +5,7 @@ project("ggml" C CXX ASM)
 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
 set(GGML_VERSION_MINOR 11)
-set(GGML_VERSION_PATCH 1)
+set(GGML_VERSION_PATCH 0)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")

 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
@@ -249,7 +249,6 @@ option(GGML_SYCL                            "ggml: use SYCL"
 option(GGML_SYCL_F16                        "ggml: use 16 bit floats for sycl calculations"   OFF)
 option(GGML_SYCL_GRAPH                      "ggml: enable graphs in the SYCL backend"         ON)
 option(GGML_SYCL_HOST_MEM_FALLBACK          "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON)
-option(GGML_SYCL_SUPPORT_LEVEL_ZERO         "ggml: use Level Zero API in SYCL backend"  ON)
 option(GGML_SYCL_DNN                        "ggml: enable oneDNN in the SYCL backend"         ON)
 set   (GGML_SYCL_TARGET "INTEL" CACHE STRING
                                            "ggml: sycl target device")
@@ -169,7 +169,7 @@ extern "C" {
        // device type
        enum ggml_backend_dev_type type;
        // device id
-        //   for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0")
+        //   for PCI devices, this should be the PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:01:00.0")
        //   if the id is unknown, this should be NULL
        const char * device_id;
        // device capabilities
@@ -965,7 +965,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str
        }
        if (sched->debug > 1) {
            ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);
-            GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_desc(node), node->name,
+            GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_name(node->op), node->name,
                fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node),
                graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)], node->flags & GGML_TENSOR_FLAG_COMPUTE ? 1 : 0);
            for (int j = 0; j < GGML_MAX_SRC; j++) {
@@ -450,22 +450,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
            ggml-cpu/arch/riscv/repack.cpp
            )
        if (GGML_CPU_RISCV64_SPACEMIT)
-            include(ggml-cpu/cmake/FindSMTIME.cmake)
            target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_RISCV64_SPACEMIT ${RISCV64_SPACEMIT_IME_SPEC})
            list(APPEND GGML_CPU_SOURCES
                ggml-cpu/spacemit/ime.cpp
                ggml-cpu/spacemit/ime.h
-                ggml-cpu/spacemit/spine_mem_pool.cpp
-                ggml-cpu/spacemit/spine_mem_pool.h
-                ggml-cpu/spacemit/repack.cpp
-                ggml-cpu/spacemit/repack.h
-                ggml-cpu/spacemit/ime_env.cpp
-                ggml-cpu/spacemit/ime_env.h
                ggml-cpu/spacemit/ime1_kernels.cpp
-                ggml-cpu/spacemit/ime2_kernels.cpp
                ggml-cpu/spacemit/ime_kernels.h
-                ggml-cpu/spacemit/rvv_kernels.cpp
-                ggml-cpu/spacemit/rvv_kernels.h
            )
        endif()
        if(NOT GGML_CPU_ALL_VARIANTS)
@@ -495,9 +485,6 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
            if (GGML_RV_ZIHINTPAUSE)
                string(APPEND MARCH_STR "_zihintpause")
            endif()
-            if (GGML_RV_ZBA)
-                string(APPEND MARCH_STR "_zba")
-            endif()
            if (GGML_CPU_RISCV64_SPACEMIT)
                # `xsmtvdotii' is only required for GCC >= 15.
                if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND
@@ -1,32 +0,0 @@
-include(CheckCSourceRuns)
-
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv)" AND GGML_CPU_RISCV64_SPACEMIT)
-    set(SMT_MARCH_STR "-march=rv64gcv_zfh_zvfh_zba_zicbop")
-    if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND
-        CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 15)
-        string(APPEND SMT_MARCH_STR "_xsmtvdotii")
-    endif()
-    set(CMAKE_REQUIRED_FLAGS "${SMT_MARCH_STR}")
-
-    check_c_source_compiles("int main() {__asm__ volatile(\"vmadot v2, v0, v1\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_IME1)
-    check_c_source_compiles("int main() {__asm__ volatile(\"vmadot v2, v0, v1, i4\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VMADOT_S4)
-    check_c_source_compiles("int main() {__asm__ volatile(\"vmadot v2, v0, v1, i8\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VMADOT_S8)
-    check_c_source_compiles("int main() {__asm__ volatile(\"vfwmadot v2, v0, v1, fp16\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VFWMADOT_FP16)
-    check_c_source_compiles("int main() {__asm__ volatile(\"vmadot.hp v2, v0, v1, v0, 0, i4\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VFMADOT_S4)
-    check_c_source_compiles("int main() {__asm__ volatile(\"vmadot.hp v2, v0, v1, v0, 0, i8\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VFMADOT_S8)
-    check_c_source_compiles("int main() {__asm__ volatile(\"vmadot1 v2, v0, v1\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VMADOTN)
-    check_c_source_compiles("int main() {__asm__ volatile(\"vpack.vv v2, v0, v1, 2\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VPACK)
-    check_c_source_compiles("int main() {__asm__ volatile(\"vnspack.vv v2, v0, v1, 2\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VNPACK)
-    unset(CMAKE_REQUIRED_FLAGS)
-
-    list(APPEND RISCV64_SPACEMIT_IME_SPEC "")
-    if (SPACEMIT_RISCV_COMPILER_SUPPORT_IME1)
-        set(RISCV64_SPACEMIT_IME_SPEC "RISCV64_SPACEMIT_IME1")
-    endif()
-
-    if (SPACEMIT_RISCV_COMPILER_SUPPORT_VMADOT_S4 AND SPACEMIT_RISCV_COMPILER_SUPPORT_VPACK AND SPACEMIT_RISCV_COMPILER_SUPPORT_VNPACK)
-        list(APPEND RISCV64_SPACEMIT_IME_SPEC "RISCV64_SPACEMIT_IME2")
-    endif()
-
-    message("RISCV64_SPACEMIT_IME_SPEC: ${RISCV64_SPACEMIT_IME_SPEC}")
-endif()
@@ -50,10 +50,6 @@
 #include "llamafile/sgemm.h"
 #endif

-#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
-#    include "spacemit/ime.h"
-#endif
-
 // Note: once we move threading into a separate C++ file
 // will use std::hardware_destructive_interference_size instead of hardcoding it here
 // and we'll use C++ attribute syntax.
@@ -3015,11 +3011,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
    const struct ggml_cgraph * cgraph = tp->cgraph;
    const struct ggml_cplan  * cplan  = tp->cplan;

-#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
-    ggml_backend_cpu_riscv64_spacemit_set_numa_thread_affinity(state->ith);
-#else
    set_numa_thread_affinity(state->ith);
-#endif

    struct ggml_compute_params params = {
        /*.ith        =*/ state->ith,
@@ -3076,10 +3068,6 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {

    ggml_barrier(state->threadpool);

-#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
-    ggml_backend_cpu_riscv64_spacemit_clear_numa_thread_affinity_threaded(state->ith);
-#endif
-
    return 0;
 }

@@ -8,14 +8,6 @@ extern "C" {

 ggml_backend_buffer_type_t ggml_backend_cpu_riscv64_spacemit_buffer_type(void);

-void ggml_backend_cpu_riscv64_spacemit_set_numa_thread_affinity(int thread_n);
-
-void ggml_backend_cpu_riscv64_spacemit_clear_numa_thread_affinity_threaded(int thread_n);
-
-void * ggml_backend_cpu_riscv64_spacemit_alloc_shared(size_t size, size_t alignment);
-
-void ggml_backend_cpu_riscv64_spacemit_free_shared(void * ptr);
-
 #ifdef __cplusplus
 }
 #endif
@@ -1,320 +0,0 @@
-#include "ime_env.h"
-
-#include "ggml-impl.h"
-#include "spine_mem_pool.h"
-
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <algorithm>
-#include <array>
-#include <cctype>
-#include <fstream>
-#include <string>
-#include <thread>
-#include <unordered_map>
-
-namespace ggml::cpu::riscv64_spacemit {
-bool spine_core_info::get_spine_core_info(std::vector<spine_core_info> & result) {
-    static std::unordered_map<uint64_t, spine_core_arch_id> spine_march_mapping_ = {
-        {0x8000000058000001,  spine_core_arch_id::core_arch_x60 },
-        { 0x8000000041000001, spine_core_arch_id::core_arch_a60 },
-        { 0x8000000058000002, spine_core_arch_id::core_arch_x100},
-        { 0x8000000041000002, spine_core_arch_id::core_arch_a100},
-    };
-
-    result.clear();
-    std::ifstream file("/proc/cpuinfo");
-    std::string   line;
-
-    std::vector<std::array<uint64_t, 2>> cpu_info_list;
-
-    uint64_t current_processor = spine_invalid_core_id;
-    uint64_t current_marchid   = 0;
-    bool     has_processor     = false;
-    bool     has_marchid       = false;
-
-    if (!file.is_open()) {
-        return false;
-    }
-
-    while (std::getline(file, line)) {
-        if (line.substr(0, 9) == "processor") {
-            if (has_processor && has_marchid) {
-                cpu_info_list.push_back({ current_processor, current_marchid });
-            }
-
-            size_t colon_pos = line.find(':');
-            if (colon_pos != std::string::npos) {
-                current_processor = std::stoi(line.substr(colon_pos + 1));
-                has_processor     = true;
-            }
-
-            has_marchid = false;
-        } else if (line.substr(0, 7) == "marchid") {
-            size_t colon_pos = line.find(':');
-            if (colon_pos != std::string::npos) {
-                std::string marchid_str = line.substr(colon_pos + 1);
-                marchid_str.erase(std::remove_if(marchid_str.begin(), marchid_str.end(), isspace), marchid_str.end());
-                current_marchid = std::stoull(marchid_str, nullptr, 16);
-                has_marchid     = true;
-            }
-        }
-    }
-
-    if (has_processor && has_marchid) {
-        cpu_info_list.push_back({ current_processor, current_marchid });
-    }
-
-    if (has_processor && has_marchid) {
-        for (auto & cpu_info : cpu_info_list) {
-            if (cpu_info[0] != spine_invalid_core_id &&
-                spine_march_mapping_.find(cpu_info[1]) != spine_march_mapping_.end()) {
-                auto core_info    = spine_core_info();
-                core_info.core_id = cpu_info[0];
-                core_info.arch_id = spine_core_arch_id(spine_march_mapping_[cpu_info[1]]);
-
-                result.push_back(core_info);
-            }
-        }
-    }
-
-    return has_processor && has_marchid;
-}
-
-namespace {
-uint16_t hex_string_to_u16(const std::string & hex_str) {
-    try {
-        size_t pos = 0;
-        if (hex_str.substr(0, 2) == "0x" || hex_str.substr(0, 2) == "0X") {
-            pos = 2;
-        }
-        unsigned long result = std::stoul(hex_str.substr(pos), nullptr, 16);
-        if (result > std::numeric_limits<uint16_t>::max()) {
-            throw std::out_of_range("Converted value is out of range for uint16_t");
-        }
-        return static_cast<uint16_t>(result);
-    } catch (const std::invalid_argument & e) {
-        throw std::invalid_argument("Invalid hexadecimal string");
-    } catch (const std::out_of_range & e) {
-        throw;
-    }
-}
-
-const char * spine_mem_pool_backend_to_string(spine_mem_pool_backend backend) {
-    switch (backend) {
-        case spine_mem_pool_backend::none:
-            return "NONE";
-        case spine_mem_pool_backend::posix_memalign:
-            return "POSIX";
-        case spine_mem_pool_backend::transparent_hugepage:
-            return "HPAGE";
-        case spine_mem_pool_backend::hugetlb_1g:
-            return "HPAGE1GB";
-    }
-
-    return "unknown";
-}
-
-spine_mem_pool_backend parse_mem_backend(const char * mem_backend_str) {
-    if (mem_backend_str == nullptr || mem_backend_str[0] == '\0') {
-        return spine_mem_pool_backend::transparent_hugepage;
-    }
-
-    std::string value(mem_backend_str);
-    std::transform(value.begin(), value.end(), value.begin(),
-                   [](unsigned char ch) { return static_cast<char>(std::tolower(ch)); });
-
-    if (value == "none") {
-        return spine_mem_pool_backend::none;
-    }
-
-    if (value == "posix") {
-        return spine_mem_pool_backend::posix_memalign;
-    }
-
-    if (value == "hpage") {
-        return spine_mem_pool_backend::transparent_hugepage;
-    }
-
-    if (value == "hpage1gb") {
-        return spine_mem_pool_backend::hugetlb_1g;
-    }
-
-    throw std::runtime_error("invalid SPACEMIT_MEM_BACKEND: " + value + ", expected NONE, POSIX, HPAGE or HPAGE1GB");
-}
-}  // namespace
-
-spine_env_info::spine_env_info() {
-    num_cores = static_cast<int>(std::thread::hardware_concurrency());
-    spine_core_info::get_spine_core_info(core_info_list);
-
-    // special for x60 K1
-    if (core_info_list.size() == 8 && core_info_list[0].arch_id == spine_core_arch_id::core_arch_x60) {
-        for (int i = 0; i < 4; i++) {
-            core_info_list[i].arch_id = spine_core_arch_id::core_arch_a60;
-        }
-    }
-
-    // special for qemu
-    if (core_info_list.size() == 0) {
-        char * spine_core_arch_str = getenv("SPACEMIT_CORE_ARCH");
-        if (spine_core_arch_str != nullptr) {
-            auto arch_id = hex_string_to_u16(spine_core_arch_str);
-            for (int i = 0; i < num_cores; i++) {
-                auto core_info    = spine_core_info();
-                core_info.core_id = i;
-                core_info.arch_id = spine_core_arch_id{ arch_id };
-                core_info_list.push_back(core_info);
-            }
-        }
-    }
-
-    if (core_info_list.size() == 0) {
-        throw std::runtime_error(
-            "Failed to get SPACEMIT_CORE_ARCH from environment or failed to parse it from /proc/cpuinfo");
-    }
-
-    char * spine_perfer_core_arch_str = getenv("SPACEMIT_PERFER_CORE_ARCH");
-    if (spine_perfer_core_arch_str != nullptr && spine_perfer_core_arch_str != "") {
-        perfer_core_arch_id = spine_core_arch_id{ hex_string_to_u16(spine_perfer_core_arch_str) };
-    }
-
-    char *           spine_perfer_core_id_str = getenv("SPACEMIT_PERFER_CORE_ID");
-    std::vector<int> perfer_core_id_vec;
-    if (spine_perfer_core_id_str != nullptr && spine_perfer_core_id_str != "") {
-        std::string perfer_core_id_str(spine_perfer_core_id_str);
-        size_t      start = 0;
-        size_t      end   = 0;
-        while ((end = perfer_core_id_str.find(',', start)) != std::string::npos) {
-            std::string core_id_substr = perfer_core_id_str.substr(start, end - start);
-            perfer_core_id_vec.push_back(std::stoi(core_id_substr));
-            start = end + 1;
-        }
-        std::string core_id_substr = perfer_core_id_str.substr(start);
-        perfer_core_id_vec.push_back(std::stoi(core_id_substr));
-    }
-
-    perfer_core_ids.reserve(num_cores);
-    if (perfer_core_arch_id == spine_core_arch_id::core_arch_none) {
-        for (auto & core_info : core_info_list) {
-            auto core_arch_id   = core_info.arch_id;
-            auto core_arch_head = (uint16_t) (core_arch_id) >> 12;
-            if (core_arch_head == 0xA) {
-                num_perfer_cores++;
-                perfer_core_arch_id = core_arch_id;
-                cpu_mask |= (1ULL << core_info.core_id);
-                perfer_core_ids.push_back(core_info.core_id);
-            }
-        }
-    } else {
-        for (auto & core_info : core_info_list) {
-            auto core_arch_id = core_info.arch_id;
-            if (core_arch_id == perfer_core_arch_id) {
-                num_perfer_cores++;
-                cpu_mask |= (1ULL << core_info.core_id);
-
-                auto core_arch_head = (uint16_t) (core_arch_id) >> 12;
-                if (core_arch_head == 0xA) {
-                    perfer_core_ids.push_back(core_info.core_id);
-                }
-            }
-        }
-        if (num_perfer_cores == 0) {
-            GGML_ABORT("can not find core with arch id %x for SPACEMIT_PERFER_CORE_ARCH in core info list\n",
-                       (uint16_t) perfer_core_arch_id);
-        }
-    }
-
-    if (perfer_core_id_vec.size() > 0) {
-        perfer_core_ids.clear();
-        cpu_mask         = 0;
-        num_perfer_cores = 0;
-        for (int core_id : perfer_core_id_vec) {
-            if (core_id < 0 || core_id >= num_cores) {
-                GGML_ABORT("invalid core id in SPACEMIT_PERFER_CORE_ID: %d, should be between 0 and %d\n", core_id,
-                           num_cores - 1);
-            }
-            auto core_info    = core_info_list[core_id];
-            auto core_arch_id = core_info.arch_id;
-            if (core_arch_id == perfer_core_arch_id) {
-                cpu_mask |= (1ULL << core_id);
-                perfer_core_ids.push_back(core_id);
-            } else {
-                GGML_ABORT(
-                    "core id %d in SPACEMIT_PERFER_CORE_ID has arch id %x which does not match "
-                    "SPACEMIT_PERFER_CORE_ARCH %x\n",
-                    core_id, (uint16_t) core_arch_id, (uint16_t) perfer_core_arch_id);
-            }
-        }
-        std::string perfer_core_id_vec_str;
-        for (int core_id : perfer_core_id_vec) {
-            perfer_core_id_vec_str += std::to_string(core_id) + ",";
-        }
-        perfer_core_id_vec_str.pop_back();
-        GGML_LOG_DEBUG("SPACEMIT_PERFER_CORE_ID is set, perferred core ids: %s\n", perfer_core_id_vec_str.c_str());
-        num_perfer_cores = static_cast<int>(perfer_core_id_vec.size());
-    }
-
-    use_ime1 = perfer_core_arch_id == spine_core_arch_id::core_arch_a60 ||
-               perfer_core_arch_id == spine_core_arch_id::core_arch_x100;
-
-    use_ime2 = perfer_core_arch_id == spine_core_arch_id::core_arch_a100;
-
-    mem_backend                  = parse_mem_backend(getenv("SPACEMIT_MEM_BACKEND"));
-    char * spine_disable_tcm_str = getenv("SPACEMIT_DISABLE_TCM");
-    auto   user_disable_tcm      = spine_disable_tcm_str != nullptr && strcmp(spine_disable_tcm_str, "0") != 0;
-
-    if (!user_disable_tcm) {
-        spine_mem_pool_tcm_info tcm_info;
-        if (spine_mem_pool_tcm_init(&tcm_info)) {
-            use_tcm      = tcm_info.available;
-            tcm_blk_size = tcm_info.blk_size;
-            GGML_LOG_DEBUG("CPU_RISCV64_SPACEMIT: tcm is available, blk_size: %zu, blk_num: %zu, is_fake_tcm: %d\n",
-                           tcm_info.blk_size, tcm_info.blk_num, tcm_info.is_fake_tcm);
-
-            for (auto & core_info : core_info_list) {
-                auto core_arch_head = (uint16_t) (core_info.arch_id) >> 12;
-                if (core_arch_head != 0xA) {
-                    aicpu_id_offset++;
-                } else {
-                    break;
-                }
-            }
-        }
-    }
-
-    GGML_LOG_DEBUG(
-        "CPU_RISCV64_SPACEMIT: num_cores: %d, num_perfer_cores: %d, perfer_core_arch_id: %x, exclude_main_thread: %d, "
-        "use_ime1: %d, use_ime2: %d, mem_backend: %s, cpu_mask: %lx, aicpu_id_offset: %d\n",
-        num_cores, num_perfer_cores, (uint16_t) perfer_core_arch_id, exclude_main_thread, use_ime1, use_ime2,
-        spine_mem_pool_backend_to_string(mem_backend), cpu_mask, aicpu_id_offset);
-
-    const size_t init_barrier_size = sizeof(spine_barrier_t) * spine_init_barrier_count;
-    init_barrier =
-        static_cast<spine_barrier_t *>(spine_mem_pool_shared_mem_alloc(init_barrier_size, alignof(spine_barrier_t)));
-    if (init_barrier != nullptr) {
-        init_barrier_is_shared_mem = true;
-    } else {
-        GGML_LOG_WARN("CPU_RISCV64_SPACEMIT: failed to allocate init_barrier from shared mem, falling back to heap\n",
-                      __func__);
-        init_barrier = new spine_barrier_t[spine_init_barrier_count];
-    }
-
-    spine_barrier_init(init_barrier, spine_init_barrier_count, 2);
-}
-
-spine_env_info::~spine_env_info() {
-    if (init_barrier_is_shared_mem) {
-        spine_mem_pool_shared_mem_free(init_barrier);
-    } else {
-        delete[] init_barrier;
-    }
-
-    init_barrier               = nullptr;
-    init_barrier_is_shared_mem = false;
-}
-
-spine_env_info global_spine_env_info;
-
-}  // namespace ggml::cpu::riscv64_spacemit
@@ -1,55 +0,0 @@
-#pragma once
-
-#include "spine_barrier.h"
-#include "spine_mem_pool.h"
-
-#include <cstddef>
-#include <cstdint>
-#include <vector>
-
-namespace ggml::cpu::riscv64_spacemit {
-
-constexpr uint64_t spine_invalid_core_id    = 0xFFFFFFFF;
-constexpr size_t   spine_init_barrier_count = 16;
-
-enum class spine_core_arch_id : uint16_t {
-    core_arch_none = 0,
-    core_arch_x60  = 0x503C,
-    core_arch_x100 = 0x5064,
-    core_arch_x200 = 0x50C8,
-    core_arch_a60  = 0xA03C,
-    core_arch_a100 = 0xA064,
-    core_arch_a200 = 0xA0C8,
-};
-
-struct spine_core_info {
-    uint64_t           core_id{ spine_invalid_core_id };
-    spine_core_arch_id arch_id{ spine_core_arch_id::core_arch_none };
-
-    static bool get_spine_core_info(std::vector<spine_core_info> & result);
-};
-
-struct spine_env_info {
-    std::vector<spine_core_info> core_info_list;
-    std::vector<int>             perfer_core_ids;
-    int                          aicpu_id_offset{ 0 };
-    int                          num_cores{ 0 };
-    int                          num_perfer_cores{ 0 };
-    spine_core_arch_id           perfer_core_arch_id{ spine_core_arch_id::core_arch_none };
-    bool                         exclude_main_thread{ false };
-    bool                         use_ime2{ false };
-    bool                         use_ime1{ false };
-    bool                         use_tcm{ false };
-    spine_mem_pool_backend       mem_backend{ spine_mem_pool_backend::transparent_hugepage };
-    uint64_t                     tcm_blk_size{ 0 };
-    uint64_t                     cpu_mask{ 0 };
-    spine_barrier_t *            init_barrier{ nullptr };
-    bool                         init_barrier_is_shared_mem{ false };
-
-    spine_env_info();
-    ~spine_env_info();
-};
-
-extern spine_env_info global_spine_env_info;
-
-}  // namespace ggml::cpu::riscv64_spacemit
@@ -1,189 +1,26 @@
 #pragma once

-#include <cassert>
 #include <cstddef>
-#include <functional>
-
-namespace spacemit_kernels {
-
-#define BLOCK_QNK_LEN 256
-
-template <int N> struct nrow_block_q2_k {
-    // [4bit scale + 4bit zp] * N * 16
-    uint8_t  scales[N * BLOCK_QNK_LEN / 16];
-    // [b0, b16, b32, b48] [b1, b17, b33, b49] ... [b15, b31, b47, b63]
-    // [b64, b80, b96, b112] ...[b79, b95, b111, b127]
-    // [b128, b144, b160, b176] ...[b143, b159, b175, b191]
-    // [b192, b208, b224, b240] ...[b207, b223, b239, b255]
-    uint8_t  qs[N * BLOCK_QNK_LEN / 4];
-    uint16_t scales16[N];
-    uint16_t zeros16[N];
-};
-
-template <int N> struct nrow_block_q3_k {
-    // [8bit scale] * N * 16
-    int8_t   scales[N * 16];
-    // [b0, b1, b2, b3, b4, b5, b6, b7] ... [b248, b249, b250, b251, b252, b253, b254, b255]
-    uint8_t  hmask[N * BLOCK_QNK_LEN / 8];
-    // [b0, b16, b32, b48] [b1, b17, b33, b49] ... [b15, b31, b47, b63]
-    // [b64, b80, b96, b112] ...[b79, b95, b111, b127]
-    // [b128, b144, b160, b176] ...[b143, b159, b175, b191]
-    // [b192, b208, b224, b240] ...[b207, b223, b239, b255]
-    uint8_t  qs[N * BLOCK_QNK_LEN / 4];
-    uint16_t scales16[N];
-};
-
-template <int N> struct nrow_block_mxfp4 {
-    uint8_t e[N];
-    uint8_t qh[4 * N];
-    uint8_t qs[16 * N];
-};
-
-template <int N> struct __attribute__((packed)) nrow_block_q5_1 {
-    uint16_t scales16[N];
-    uint8_t  zp[N];
-    // n0 [bh0, bh1, bh2, bh3, bh4, bh5, bh6, bh7] ....
-    uint8_t  qh[4 * N];
-    // n0 [b0, b1], [b2, b3] ....  [b30, b31]
-    // n1 [b0, b1], [b2, b3] ....  [b30, b31]
-    uint8_t  qs[16 * N];
-};
-
-static_assert(sizeof(nrow_block_q5_1<1>) == sizeof(uint8_t) + 22, "wrong nrow_block_q5_1 block size/padding");
-
-template <int N> struct __attribute__((packed)) nrow_block_q5_0 {
-    uint16_t scales16[N];
-    // n0 [bh0, bh1, bh2, bh3, bh4, bh5, bh6, bh7] ....
-    uint8_t  qh[4 * N];
-    // n0 [b0, b1], [b2, b3] ....  [b30, b31]
-    // n1 [b0, b1], [b2, b3] ....  [b30, b31]
-    uint8_t  qs[16 * N];
-};
-
-static_assert(sizeof(nrow_block_q5_0<1>) == 22, "wrong nrow_block_q5_0 block size/padding");
-
-using gemm_kernel_quantize_def = std::function<
-    size_t(size_t, const uint8_t *, const uint8_t *, const uint8_t *, float *, size_t, size_t, size_t, size_t)>;
-
-using moe_gemm_kernel_quantize_def = std::function<
-    size_t(size_t, const uint8_t **, const uint8_t *, const uint8_t *, float **, size_t, size_t, size_t, size_t)>;

+namespace sqnbitgemm_spacemit_ime {
 namespace ime1 {
-size_t gemm_kernel_i8i4(size_t          blk_len,
-                        const uint8_t * quant_a_ptr,
-                        const uint8_t * quant_b_data,
-                        const uint8_t * quant_b_zp,
-                        float *         c_ptr,
-                        size_t          count_m,
-                        size_t          count_n,
-                        size_t          k_blks,
-                        size_t          ldc);
+size_t gemm_kernel_i8i4(size_t            blk_len,
+                        const std::byte * quant_a_ptr,
+                        const std::byte * quant_b_data,
+                        const float *     quant_b_scale,
+                        const std::byte * quant_b_zp,
+                        float *           c_ptr,
+                        size_t            count_m,
+                        size_t            count_n,
+                        size_t            count_k,
+                        size_t            block_count_k,
+                        size_t            ldc,
+                        const float *     bias,
+                        const size_t      scale_stride);

-void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
+void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, std::byte * quant_a_ptr);

-void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
+void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, std::byte * quant_a_ptr);

 }  // namespace ime1
-
-namespace ime2 {
-size_t gemm_kernel_i8i2k(size_t          blk_len,
-                         const uint8_t * quant_a_ptr,
-                         const uint8_t * quant_b_data,
-                         const uint8_t * quant_b_zp,
-                         float *         c_ptr,
-                         size_t          count_m,
-                         size_t          count_n,
-                         size_t          k_blks,
-                         size_t          ldc);
-
-size_t gemm_kernel_i8i3k(size_t          blk_len,
-                         const uint8_t * quant_a_ptr,
-                         const uint8_t * quant_b_data,
-                         const uint8_t * quant_b_zp,
-                         float *         c_ptr,
-                         size_t          count_m,
-                         size_t          count_n,
-                         size_t          k_blks,
-                         size_t          ldc);
-
-size_t gemm_kernel_i8i4(size_t          blk_len,
-                        const uint8_t * quant_a_ptr,
-                        const uint8_t * quant_b_data,
-                        const uint8_t * quant_b_zp,
-                        float *         c_ptr,
-                        size_t          count_m,
-                        size_t          count_n,
-                        size_t          k_blks,
-                        size_t          ldc);
-
-size_t gemm_kernel_i8i4_hp(size_t          blk_len,
-                           const uint8_t * quant_a_ptr,
-                           const uint8_t * quant_b_data,
-                           const uint8_t * quant_b_zp,
-                           float *         c_ptr,
-                           size_t          count_m,
-                           size_t          count_n,
-                           size_t          k_blks,
-                           size_t          ldc);
-
-size_t moe_m2_gemm_kernel_i8i4(size_t           blk_len,
-                               const uint8_t ** quant_a_ptr,
-                               const uint8_t *  quant_b_data,
-                               const uint8_t *  quant_b_zp,
-                               float **         c_ptr,
-                               size_t           count_m,
-                               size_t           count_n,
-                               size_t           k_blks,
-                               size_t           ldc);
-
-size_t gemm_kernel_i8i8(size_t          blk_len,
-                        const uint8_t * quant_a_ptr,
-                        const uint8_t * quant_b_data,
-                        const uint8_t * quant_b_zp,
-                        float *         c_ptr,
-                        size_t          count_m,
-                        size_t          count_n,
-                        size_t          k_blks,
-                        size_t          ldc);
-
-size_t gemm_kernel_i8mxfp4(size_t          blk_len,
-                           const uint8_t * quant_a_ptr,
-                           const uint8_t * quant_b_data,
-                           const uint8_t * quant_b_zp,
-                           float *         c_ptr,
-                           size_t          count_m,
-                           size_t          count_n,
-                           size_t          k_blks,
-                           size_t          ldc);
-
-size_t moe_m2_gemm_kernel_i8mxfp4(size_t           blk_len,
-                                  const uint8_t ** quant_a_ptr,
-                                  const uint8_t *  quant_b_data,
-                                  const uint8_t *  quant_b_zp,
-                                  float **         c_ptr,
-                                  size_t           count_m,
-                                  size_t           count_n,
-                                  size_t           k_blks,
-                                  size_t           ldc);
-
-size_t gemm_kernel_i8i5(size_t          blk_len,
-                        const uint8_t * quant_a_ptr,
-                        const uint8_t * quant_b_data,
-                        const uint8_t * quant_b_zp,
-                        float *         c_ptr,
-                        size_t          count_m,
-                        size_t          count_n,
-                        size_t          k_blks,
-                        size_t          ldc);
-
-size_t moe_m2_gemm_kernel_i8i5(size_t           blk_len,
-                               const uint8_t ** quant_a_ptr,
-                               const uint8_t *  quant_b_data,
-                               const uint8_t *  quant_b_zp,
-                               float **         c_ptr,
-                               size_t           count_m,
-                               size_t           count_n,
-                               size_t           k_blks,
-                               size_t           ldc);
-}  // namespace ime2
-}  // namespace spacemit_kernels
+}  // namespace sqnbitgemm_spacemit_ime
@@ -1,14 +0,0 @@
-#pragma once
-
-#include "ggml-common.h"
-#include "ggml.h"
-
-#include <cstddef>
-#include <cstdint>
-
-namespace ggml::cpu::riscv64_spacemit {
-
-template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS>
-int repack(ggml_tensor * t, const void * data, size_t data_size);
-
-}  // namespace ggml::cpu::riscv64_spacemit
@@ -1,95 +0,0 @@
-#pragma once
-
-#include "ggml-cpu-impl.h"
-
-#include <cassert>
-#include <cstddef>
-#include <cstdint>
-#include <functional>
-
-namespace spacemit_kernels {
-
-constexpr auto div_round_up(auto up, auto down) {
-    return (up + down - 1) / down;
-}
-
-// Q8 Blk [f32] [s16] [int8 * blk_len]
-// Q8 Blk N [f32 * N] [s16 * N] [int8 * blk_len * N]
-constexpr size_t q8_blk_size(size_t blk_len, bool with_blk_sum = false) {
-    const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t) + (with_blk_sum ? sizeof(int16_t) : 0);
-    return blk_size;
-}
-
-// Q8 HP row block: K is split into K32 subblocks.
-// Each subblock stores [f32 scale] [int8 * 32], with an optional fp16 sum trailer per subblock.
-constexpr size_t q8_hp_blk_size(size_t blk_len, bool with_blk_sum = false, bool with_blk_scale = false) {
-    const size_t subblk_count = div_round_up(blk_len, size_t(32));
-    const size_t blk_size     = blk_len * sizeof(int8_t) + subblk_count * sizeof(_Float16) +
-                            (with_blk_sum ? subblk_count * sizeof(_Float16) : 0) +
-                            (with_blk_scale ? sizeof(_Float16) : 0);
-    return blk_size;
-}
-
-// Q8K Blk [f32] [s16 * (blk_len / 16)] [int8 * blk_len]
-// Q8K Blk N [f32 * N] [s16 * (blk_len / 16) * N] [int8 * blk_len * N]
-constexpr size_t q8k_blk_size(size_t blk_len) {
-    const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t) + sizeof(int16_t) * blk_len / 16;
-    return blk_size;
-}
-
-using quantize_a_row_def = std::function<void(size_t, const float *, size_t, uint8_t *)>;
-
-namespace rvv {
-void memcpy1d(void * dst, const void * src, int64_t size);
-
-void memcpy2d(void * dst, int64_t dst_stride, const void * src, int64_t src_stride, int64_t tile_rows, int64_t size);
-
-void forward_flash_attn_ext_f16_one_chunk_vlen1024_vf16(const ggml_compute_params * params,
-                                                        ggml_tensor *               dst,
-                                                        int                         ir0,
-                                                        int                         ir1,
-                                                        void *                      tcm_buffer,
-                                                        size_t                      tcm_buffer_size);
-
-void forward_flash_attn_ext_f16_tiled_vlen1024_vf16(const ggml_compute_params * params,
-                                                    ggml_tensor *               dst,
-                                                    int                         ir0,
-                                                    int                         ir1,
-                                                    void *                      tcm_buffer,
-                                                    size_t                      tcm_buffer_size);
-
-void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op);
-
-void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op);
-
-void forward_cont_with_permute(ggml_compute_params * params, ggml_tensor * op);
-
-void forward_cpy_with_permute(ggml_compute_params * params, ggml_tensor * op);
-
-template <typename T> void forward_get_rows(ggml_compute_params * params, ggml_tensor * op);
-
-template <typename T> void forward_concat(ggml_compute_params * params, ggml_tensor * op);
-
-template <ggml_op op_type, typename T> void forward_binary(ggml_compute_params * params, ggml_tensor * op);
-
-template <typename T> void forward_sum_rows(const ggml_compute_params * params, ggml_tensor * op);
-
-template <typename T> void forward_repeat_nrows(ggml_compute_params * params, ggml_tensor * op);
-
-template <typename T> void forward_repeat_dim1(ggml_compute_params * params, ggml_tensor * op);
-
-void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
-
-void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
-
-void quantize_a_row_i8_hp(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
-
-void quantize_a_4row_i8_hp(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
-
-void quantize_a_row_i8k(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
-
-void quantize_a_4row_i8k(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
-
-}  // namespace rvv
-
-}  // namespace spacemit_kernels
@@ -1,34 +0,0 @@
-#pragma once
-
-#include <atomic>
-#include <cstdint>
-
-#define SPINE_CACHE_LINE  64
-#define SPINE_CACHE_ALIGN __attribute__((aligned(SPINE_CACHE_LINE)))
-
-struct spine_barrier_t {
-    SPINE_CACHE_ALIGN std::atomic<int64_t> pending_;
-    SPINE_CACHE_ALIGN std::atomic<int64_t> rounds_;
-    SPINE_CACHE_ALIGN int64_t              total_;
-};
-
-inline void spine_barrier_wait(spine_barrier_t * b) {
-    auto cur_round = b->rounds_.load(std::memory_order_acquire);
-    auto cnt       = --b->pending_;
-    if (cnt == 0) {
-        b->pending_.store(b->total_);
-        b->rounds_.store(cur_round + 1);
-    } else {
-        while (cur_round == b->rounds_.load(std::memory_order_relaxed)) {
-            __asm__ volatile("pause " ::: "memory");
-        }
-    }
-}
-
-inline void spine_barrier_init(spine_barrier_t * b, int num_barriers, uint64_t thread_count) {
-    for (int i = 0; i < num_barriers; i++) {
-        b[i].total_ = thread_count;
-        b[i].pending_.store(thread_count);
-        b[i].rounds_.store(0);
-    }
-}
@@ -1,760 +0,0 @@
-#include "spine_mem_pool.h"
-
-#include "common.h"
-#include "ime_env.h"
-#include "spine_tcm.h"
-
-#include <fcntl.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <unistd.h>
-
-#include <algorithm>
-#include <cerrno>
-#include <cstdint>
-#include <cstdlib>
-#include <limits>
-#include <memory>
-#include <mutex>
-#include <unordered_map>
-#include <vector>
-
-namespace ggml::cpu::riscv64_spacemit {
-namespace {
-
-constexpr size_t   SPINE_MEM_POOL_CHUNK_SIZE         = 512ull * 1024ull * 1024ull;
-constexpr size_t   SPINE_SHARE_MEM_POOL_CHUNK_SIZE   = 512ull * 1024ull;
-constexpr size_t   SPINE_MEM_POOL_1G_REGION_SIZE     = 1ull << 30;
-constexpr uint64_t HUGETLB_1G_FLAG_REQUIRE_PUD       = 1ull << 0;
-constexpr char     SPINE_MEM_POOL_HUGETLB_1G_DEV[]   = "/dev/hugetlb_1g";
-constexpr char     SPINE_MEM_POOL_TCM_SYNC_MEM_DEV[] = "/dev/tcm_sync_mem";
-
-struct hugetlb_1g_region {
-    uint64_t size{ 0 };
-    uint64_t dma_addr{ 0 };
-    uint64_t flags{ 0 };
-    uint64_t reserved{ 0 };
-};
-
-#define HUGETLB_1G_IOC_MAGIC 'M'
-#define HUGETLB_1G_IOC_ALLOC _IOWR(HUGETLB_1G_IOC_MAGIC, 0x00, struct hugetlb_1g_region)
-#define HUGETLB_1G_IOC_FREE  _IO(HUGETLB_1G_IOC_MAGIC, 0x01)
-
-struct free_block {
-    size_t offset{ 0 };
-    size_t size{ 0 };
-};
-
-struct pool_chunk {
-    uint8_t *               base{ nullptr };
-    size_t                  size{ 0 };
-    int                     fd{ -1 };
-    std::vector<free_block> free_blocks;
-};
-
-struct pool_allocation {
-    void * chunk_base{ nullptr };
-    size_t chunk_size{ 0 };
-    void * base{ nullptr };
-    size_t size{ 0 };
-};
-
-bool is_power_of_two(size_t value) {
-    return value != 0 && (value & (value - 1)) == 0;
-}
-
-bool align_up(size_t value, size_t alignment, size_t * aligned_value) {
-    if (aligned_value == nullptr || alignment == 0) {
-        return false;
-    }
-
-    const size_t remainder = value % alignment;
-    if (remainder == 0) {
-        *aligned_value = value;
-        return true;
-    }
-
-    const size_t padding = alignment - remainder;
-    if (value > std::numeric_limits<size_t>::max() - padding) {
-        return false;
-    }
-
-    *aligned_value = value + padding;
-    return true;
-}
-
-bool align_up_uintptr(uintptr_t value, size_t alignment, uintptr_t * aligned_value) {
-    if (aligned_value == nullptr || alignment == 0) {
-        return false;
-    }
-
-    const uintptr_t remainder = value % alignment;
-    if (remainder == 0) {
-        *aligned_value = value;
-        return true;
-    }
-
-    const uintptr_t padding = alignment - remainder;
-    if (value > std::numeric_limits<uintptr_t>::max() - padding) {
-        return false;
-    }
-
-    *aligned_value = value + padding;
-    return true;
-}
-
-class spine_mem_pool_manager {
-  public:
-    explicit spine_mem_pool_manager(size_t default_chunk_size) : default_chunk_size_(default_chunk_size) {}
-
-    virtual ~spine_mem_pool_manager() = default;
-
-    void * alloc(size_t size, size_t alignment) {
-        if (size == 0 || !is_power_of_two(alignment)) {
-            return nullptr;
-        }
-
-        size_t aligned_size = 0;
-        if (!align_up(size, alignment, &aligned_size)) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: align_up failed for size %zu alignment %zu\n", __func__, size,
-                           alignment);
-            return nullptr;
-        }
-
-        pool_allocation allocation;
-
-        std::lock_guard<std::mutex> lock(mutex_);
-
-        if (!try_alloc_locked(aligned_size, alignment, &allocation)) {
-            if (!add_chunk_locked(aligned_size, alignment)) {
-                return nullptr;
-            }
-
-            if (!try_alloc_locked(aligned_size, alignment, &allocation)) {
-                GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: allocation retry failed for size %zu alignment %zu\n",
-                               __func__, aligned_size, alignment);
-                return nullptr;
-            }
-        }
-
-        try {
-            const auto [allocation_it, inserted] = allocations_.emplace(allocation.base, allocation);
-            if (!inserted) {
-                GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: duplicate allocation key %p\n", __func__, allocation.base);
-                rollback_allocation_locked(allocation);
-                return nullptr;
-            }
-        } catch (const std::bad_alloc &) {
-            rollback_allocation_locked(allocation);
-            throw;
-        }
-
-        return allocation.base;
-    }
-
-    void free(void * base) {
-        if (base == nullptr) {
-            return;
-        }
-
-        std::lock_guard<std::mutex> lock(mutex_);
-
-        auto allocation_it = allocations_.find(base);
-        if (allocation_it == allocations_.end()) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: unknown allocation %p\n", __func__, base);
-            return;
-        }
-
-        pool_allocation allocation = allocation_it->second;
-        allocations_.erase(allocation_it);
-
-        auto chunk_it = find_chunk_locked(allocation);
-        if (chunk_it == chunks_.end()) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: unknown chunk for allocation %p size %zu\n", __func__,
-                           allocation.base, allocation.size);
-            return;
-        }
-
-        auto * chunk_base = chunk_it->base;
-        auto * alloc_base = static_cast<uint8_t *>(allocation.base);
-        if (alloc_base < chunk_base || alloc_base >= chunk_base + chunk_it->size) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: allocation %p out of chunk range %p..%p\n", __func__,
-                           allocation.base, chunk_base, chunk_base + chunk_it->size);
-            return;
-        }
-
-        const size_t offset = static_cast<size_t>(alloc_base - chunk_base);
-        if (offset > chunk_it->size || allocation.size > chunk_it->size - offset) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: allocation %p size %zu exceeds chunk size %zu\n", __func__,
-                           allocation.base, allocation.size, chunk_it->size);
-            return;
-        }
-
-        insert_free_block_locked(*chunk_it, { offset, allocation.size });
-        maybe_release_empty_chunk_locked(chunk_it);
-    }
-
-  protected:
-    void release_chunks() {
-        std::lock_guard<std::mutex> lock(mutex_);
-
-        allocations_.clear();
-        for (auto & chunk : chunks_) {
-            dealloc_chunk(&chunk);
-        }
-        chunks_.clear();
-    }
-
-    size_t default_chunk_size() const { return default_chunk_size_; }
-
-    static void clear_chunk(pool_chunk * chunk) {
-        chunk->base = nullptr;
-        chunk->size = 0;
-        chunk->fd   = -1;
-        chunk->free_blocks.clear();
-    }
-
-    virtual bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) = 0;
-    virtual void dealloc_chunk(pool_chunk * chunk)                                                    = 0;
-
-  private:
-    struct alloc_candidate {
-        size_t    chunk_index{ 0 };
-        size_t    block_index{ 0 };
-        size_t    aligned_offset{ 0 };
-        uintptr_t address{ std::numeric_limits<uintptr_t>::max() };
-        bool      valid{ false };
-    };
-
-    std::vector<pool_chunk>::iterator find_chunk_locked(const pool_allocation & allocation) {
-        return std::find_if(chunks_.begin(), chunks_.end(), [&](const pool_chunk & chunk) {
-            return chunk.base == allocation.chunk_base && chunk.size == allocation.chunk_size;
-        });
-    }
-
-    bool add_chunk_locked(size_t min_size, size_t alignment) {
-        pool_chunk   chunk;
-        const size_t chunk_request = default_chunk_size_ == 0 ? min_size : std::max(min_size, default_chunk_size_);
-        void *       hint_addr     = nullptr;
-
-        for (const auto & existing_chunk : chunks_) {
-            auto * chunk_end = existing_chunk.base + existing_chunk.size;
-            if (hint_addr == nullptr || chunk_end > hint_addr) {
-                hint_addr = chunk_end;
-            }
-        }
-
-        if (!alloc_chunk(chunk_request, alignment, hint_addr, &chunk)) {
-            return false;
-        }
-
-        if (chunk.base == nullptr || chunk.size < min_size) {
-            GGML_LOG_ERROR(
-                "CPU_RISCV64_SPACEMIT: %s: invalid chunk returned for request size %zu, chunk_base=%p chunk_size=%zu\n",
-                __func__, min_size, chunk.base, chunk.size);
-            dealloc_chunk(&chunk);
-            return false;
-        }
-
-        try {
-            chunk.free_blocks.push_back({ 0, chunk.size });
-            chunks_.push_back(std::move(chunk));
-        } catch (const std::bad_alloc &) {
-            dealloc_chunk(&chunk);
-            throw;
-        }
-
-        return true;
-    }
-
-    void rollback_allocation_locked(const pool_allocation & allocation) {
-        auto chunk_it = find_chunk_locked(allocation);
-        if (chunk_it == chunks_.end()) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to rollback allocation %p, owning chunk not found\n",
-                           __func__, allocation.base);
-            return;
-        }
-
-        auto * chunk_base = chunk_it->base;
-        auto * alloc_base = static_cast<uint8_t *>(allocation.base);
-        if (alloc_base < chunk_base || alloc_base >= chunk_base + chunk_it->size) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to rollback allocation %p, chunk range is invalid\n",
-                           __func__, allocation.base);
-            return;
-        }
-
-        const size_t offset = static_cast<size_t>(alloc_base - chunk_base);
-        if (offset > chunk_it->size || allocation.size > chunk_it->size - offset) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to rollback allocation %p size %zu\n", __func__,
-                           allocation.base, allocation.size);
-            return;
-        }
-
-        insert_free_block_locked(*chunk_it, { offset, allocation.size });
-        maybe_release_empty_chunk_locked(chunk_it);
-    }
-
-    bool try_alloc_locked(size_t size, size_t alignment, pool_allocation * allocation) {
-        alloc_candidate best;
-
-        for (size_t chunk_index = 0; chunk_index < chunks_.size(); ++chunk_index) {
-            const auto & chunk = chunks_[chunk_index];
-            for (size_t block_index = 0; block_index < chunk.free_blocks.size(); ++block_index) {
-                const auto & block = chunk.free_blocks[block_index];
-
-                uintptr_t  aligned_addr = 0;
-                const auto block_addr   = reinterpret_cast<uintptr_t>(chunk.base + block.offset);
-                if (!align_up_uintptr(block_addr, alignment, &aligned_addr)) {
-                    continue;
-                }
-
-                if (aligned_addr < block_addr) {
-                    continue;
-                }
-
-                const size_t aligned_offset = block.offset + static_cast<size_t>(aligned_addr - block_addr);
-                const size_t padding        = aligned_offset - block.offset;
-                if (padding > block.size || size > block.size - padding) {
-                    continue;
-                }
-
-                if (!best.valid || aligned_addr < best.address) {
-                    best.chunk_index    = chunk_index;
-                    best.block_index    = block_index;
-                    best.aligned_offset = aligned_offset;
-                    best.address        = aligned_addr;
-                    best.valid          = true;
-                }
-            }
-        }
-
-        if (!best.valid) {
-            return false;
-        }
-
-        auto &           chunk     = chunks_[best.chunk_index];
-        const free_block block     = chunk.free_blocks[best.block_index];
-        const size_t     padding   = best.aligned_offset - block.offset;
-        const size_t     alloc_end = best.aligned_offset + size;
-        const size_t     block_end = block.offset + block.size;
-
-        chunk.free_blocks.erase(chunk.free_blocks.begin() + best.block_index);
-        auto insert_it = chunk.free_blocks.begin() + best.block_index;
-        if (padding != 0) {
-            insert_it = chunk.free_blocks.insert(insert_it, { block.offset, padding });
-            ++insert_it;
-        }
-        if (alloc_end < block_end) {
-            chunk.free_blocks.insert(insert_it, { alloc_end, block_end - alloc_end });
-        }
-
-        allocation->chunk_base = chunk.base;
-        allocation->chunk_size = chunk.size;
-        allocation->base       = chunk.base + best.aligned_offset;
-        allocation->size       = size;
-        return true;
-    }
-
-    void maybe_release_empty_chunk_locked(std::vector<pool_chunk>::iterator chunk_it) {
-        if (chunk_it->free_blocks.size() != 1) {
-            return;
-        }
-
-        const auto & block = chunk_it->free_blocks.front();
-        if (block.offset != 0 || block.size != chunk_it->size) {
-            return;
-        }
-
-        dealloc_chunk(&*chunk_it);
-        chunks_.erase(chunk_it);
-    }
-
-    void insert_free_block_locked(pool_chunk & chunk, free_block block) {
-        auto it = chunk.free_blocks.begin();
-        while (it != chunk.free_blocks.end() && it->offset < block.offset) {
-            ++it;
-        }
-
-        if (it != chunk.free_blocks.begin()) {
-            const auto & prev = *(it - 1);
-            if (prev.offset + prev.size > block.offset) {
-                GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: overlapping free block at offset %zu size %zu\n", __func__,
-                               block.offset, block.size);
-                return;
-            }
-        }
-
-        if (it != chunk.free_blocks.end() && block.offset + block.size > it->offset) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: overlapping next free block at offset %zu size %zu\n", __func__,
-                           block.offset, block.size);
-            return;
-        }
-
-        it = chunk.free_blocks.insert(it, block);
-
-        if (it != chunk.free_blocks.begin()) {
-            auto prev = it - 1;
-            if (prev->offset + prev->size == it->offset) {
-                it->offset = prev->offset;
-                it->size += prev->size;
-                it = chunk.free_blocks.erase(prev);
-            }
-        }
-
-        if (it + 1 != chunk.free_blocks.end() && it->offset + it->size == (it + 1)->offset) {
-            it->size += (it + 1)->size;
-            chunk.free_blocks.erase(it + 1);
-        }
-    }
-
-    std::mutex                                  mutex_;
-    std::vector<pool_chunk>                     chunks_;
-    std::unordered_map<void *, pool_allocation> allocations_;
-    size_t                                      default_chunk_size_{ 0 };
-};
-
-class spine_mem_pool_posix final : public spine_mem_pool_manager {
-  public:
-    spine_mem_pool_posix() : spine_mem_pool_manager(0) {}
-
-    ~spine_mem_pool_posix() override { release_chunks(); }
-
-  private:
-    bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) override {
-        (void) hint_addr;
-
-        const size_t alloc_alignment = std::max(alignment, sizeof(void *));
-        void *       base            = nullptr;
-        const int    rc              = posix_memalign(&base, alloc_alignment, min_size);
-        if (rc != 0) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: posix_memalign failed for size %zu alignment %zu, rc=%d\n",
-                           __func__, min_size, alloc_alignment, rc);
-            return false;
-        }
-
-        chunk->base = static_cast<uint8_t *>(base);
-        chunk->size = min_size;
-        chunk->fd   = -1;
-        return true;
-    }
-
-    void dealloc_chunk(pool_chunk * chunk) override {
-        std::free(chunk->base);
-        clear_chunk(chunk);
-    }
-};
-
-class spine_mem_pool_transparent_hugepage final : public spine_mem_pool_manager {
-  public:
-    spine_mem_pool_transparent_hugepage() : spine_mem_pool_manager(SPINE_MEM_POOL_CHUNK_SIZE) {}
-
-    ~spine_mem_pool_transparent_hugepage() override { release_chunks(); }
-
-  private:
-    bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) override {
-        (void) alignment;
-
-        size_t chunk_size = 0;
-        if (!align_up(min_size, default_chunk_size(), &chunk_size)) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to round chunk size for %zu\n", __func__, min_size);
-            return false;
-        }
-
-        void * map_addr = mmap(hint_addr, chunk_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-        if (map_addr == MAP_FAILED) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: mmap failed for chunk size %zu, errno=%d\n", __func__, chunk_size,
-                           errno);
-            return false;
-        }
-
-        if (madvise(map_addr, chunk_size, MADV_HUGEPAGE) != 0) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: madvise(MADV_HUGEPAGE) failed for chunk size %zu, errno=%d\n",
-                           __func__, chunk_size, errno);
-            munmap(map_addr, chunk_size);
-            return false;
-        }
-
-        chunk->base = static_cast<uint8_t *>(map_addr);
-        chunk->size = chunk_size;
-        chunk->fd   = -1;
-        return true;
-    }
-
-    void dealloc_chunk(pool_chunk * chunk) override {
-        if (chunk->base != nullptr && chunk->size != 0 && munmap(chunk->base, chunk->size) != 0) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: munmap failed for chunk %p size %zu, errno=%d\n", __func__,
-                           chunk->base, chunk->size, errno);
-        }
-
-        clear_chunk(chunk);
-    }
-};
-
-class spine_mem_pool_hugetlb_1g final : public spine_mem_pool_manager {
-  public:
-    spine_mem_pool_hugetlb_1g() : spine_mem_pool_manager(SPINE_MEM_POOL_1G_REGION_SIZE) {}
-
-    ~spine_mem_pool_hugetlb_1g() override { release_chunks(); }
-
-  private:
-    bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) override {
-        (void) alignment;
-        (void) hint_addr;
-
-        size_t region_size = 0;
-        if (!align_up(min_size, SPINE_MEM_POOL_1G_REGION_SIZE, &region_size)) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to round hugetlb_1g size for %zu\n", __func__, min_size);
-            return false;
-        }
-
-        const int fd = open(SPINE_MEM_POOL_HUGETLB_1G_DEV, O_RDWR);
-        if (fd < 0) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: open(%s) failed, errno=%d\n", __func__,
-                           SPINE_MEM_POOL_HUGETLB_1G_DEV, errno);
-            return false;
-        }
-
-        hugetlb_1g_region region;
-        region.size  = region_size;
-        region.flags = HUGETLB_1G_FLAG_REQUIRE_PUD;
-        if (ioctl(fd, HUGETLB_1G_IOC_ALLOC, &region) < 0) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: HUGETLB_1G_IOC_ALLOC failed for size %zu, errno=%d\n", __func__,
-                           region_size, errno);
-            close(fd);
-            return false;
-        }
-
-        void * map_addr = mmap(nullptr, region.size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-        if (map_addr == MAP_FAILED) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: mmap failed for hugetlb_1g size %llu, errno=%d\n", __func__,
-                           static_cast<unsigned long long>(region.size), errno);
-            ioctl(fd, HUGETLB_1G_IOC_FREE);
-            close(fd);
-            return false;
-        }
-
-        chunk->base = static_cast<uint8_t *>(map_addr);
-        chunk->size = region.size;
-        chunk->fd   = fd;
-        return true;
-    }
-
-    void dealloc_chunk(pool_chunk * chunk) override {
-        if (chunk->base != nullptr && chunk->size != 0 && munmap(chunk->base, chunk->size) != 0) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: munmap failed for hugetlb_1g chunk %p size %zu, errno=%d\n",
-                           __func__, chunk->base, chunk->size, errno);
-        }
-
-        if (chunk->fd >= 0) {
-            if (ioctl(chunk->fd, HUGETLB_1G_IOC_FREE) < 0) {
-                GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: HUGETLB_1G_IOC_FREE failed for chunk %p, errno=%d\n",
-                               __func__, chunk->base, errno);
-            }
-
-            close(chunk->fd);
-        }
-
-        clear_chunk(chunk);
-    }
-};
-
-class spine_mem_pool_shared_mem final : public spine_mem_pool_manager {
-  public:
-    spine_mem_pool_shared_mem() : spine_mem_pool_manager(SPINE_SHARE_MEM_POOL_CHUNK_SIZE) {}
-
-    ~spine_mem_pool_shared_mem() override { release_chunks(); }
-
-  private:
-    bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) override {
-        (void) alignment;
-
-        if (hint_addr != nullptr) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: shared_mem does not support multiple active chunks\n", __func__);
-            return false;
-        }
-
-        if (min_size > default_chunk_size()) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: shared_mem request %zu exceeds chunk size %zu\n", __func__,
-                           min_size, default_chunk_size());
-            return false;
-        }
-
-        const int fd = open(SPINE_MEM_POOL_TCM_SYNC_MEM_DEV, O_RDWR | O_SYNC);
-        if (fd < 0) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: open(%s) failed, errno=%d\n", __func__,
-                           SPINE_MEM_POOL_TCM_SYNC_MEM_DEV, errno);
-            return false;
-        }
-
-        void * map_addr = mmap(nullptr, default_chunk_size(), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-        if (map_addr == MAP_FAILED) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: mmap failed for %s size %zu, errno=%d\n", __func__,
-                           SPINE_MEM_POOL_TCM_SYNC_MEM_DEV, default_chunk_size(), errno);
-            close(fd);
-            return false;
-        }
-
-        chunk->base = static_cast<uint8_t *>(map_addr);
-        chunk->size = default_chunk_size();
-        chunk->fd   = fd;
-        return true;
-    }
-
-    void dealloc_chunk(pool_chunk * chunk) override {
-        if (chunk->base != nullptr && chunk->size != 0 && munmap(chunk->base, chunk->size) != 0) {
-            GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: munmap failed for shared_mem chunk %p size %zu, errno=%d\n",
-                           __func__, chunk->base, chunk->size, errno);
-        }
-
-        if (chunk->fd >= 0) {
-            close(chunk->fd);
-        }
-
-        clear_chunk(chunk);
-    }
-};
-
-spine_mem_pool_manager & get_spine_mem_pool_manager() {
-    static std::once_flag                          pool_once;
-    static std::unique_ptr<spine_mem_pool_manager> selected_pool;
-    static spine_mem_pool_backend                  selected_backend = spine_mem_pool_backend::none;
-
-    spine_mem_pool_backend backend = global_spine_env_info.mem_backend;
-    if (backend == spine_mem_pool_backend::none) {
-        backend = spine_mem_pool_backend::transparent_hugepage;
-    }
-
-    std::call_once(pool_once, [&]() {
-        selected_backend = backend;
-
-        switch (selected_backend) {
-            case spine_mem_pool_backend::posix_memalign:
-                selected_pool = std::make_unique<spine_mem_pool_posix>();
-                break;
-            case spine_mem_pool_backend::transparent_hugepage:
-                selected_pool = std::make_unique<spine_mem_pool_transparent_hugepage>();
-                break;
-            case spine_mem_pool_backend::hugetlb_1g:
-                selected_pool = std::make_unique<spine_mem_pool_hugetlb_1g>();
-                break;
-            case spine_mem_pool_backend::none:
-                selected_backend = spine_mem_pool_backend::transparent_hugepage;
-                selected_pool    = std::make_unique<spine_mem_pool_transparent_hugepage>();
-                break;
-        }
-    });
-
-    if (backend != selected_backend) {
-        GGML_LOG_ERROR(
-            "CPU_RISCV64_SPACEMIT: %s: mem pool backend is process-global and mutually exclusive, requested=%d but "
-            "selected=%d\n",
-            __func__, static_cast<int>(backend), static_cast<int>(selected_backend));
-    }
-
-    if (selected_pool) {
-        return *selected_pool;
-    }
-
-    throw std::bad_alloc();
-}
-
-spine_mem_pool_manager & get_spine_mem_pool_shared_mem_manager() {
-    static std::once_flag                             shared_mem_pool_once;
-    static std::unique_ptr<spine_mem_pool_shared_mem> shared_mem_pool;
-
-    std::call_once(shared_mem_pool_once, [&]() { shared_mem_pool = std::make_unique<spine_mem_pool_shared_mem>(); });
-
-    if (shared_mem_pool) {
-        return *shared_mem_pool;
-    }
-
-    throw std::bad_alloc();
-}
-
-}  // namespace
-
-bool spine_mem_pool_tcm_init(spine_mem_pool_tcm_info * info) noexcept {
-    if (info == nullptr) {
-        return false;
-    }
-
-    *info = {};
-
-    if (spine_tcm_open_handle(NULL) != 0 || !spine_tcm_is_available()) {
-        return false;
-    }
-
-    spine_tcm_mem_info_t mem_info;
-    if (spine_tcm_mem_info(&mem_info) != 0) {
-        return false;
-    }
-
-    info->available   = true;
-    info->blk_size    = mem_info.blk_size;
-    info->blk_num     = mem_info.blk_num;
-    info->is_fake_tcm = mem_info.is_fake_tcm != 0;
-    return true;
-}
-
-void * spine_mem_pool_tcm_mem_get(int cpu_id) noexcept {
-    return spine_tcm_mem_get(cpu_id);
-}
-
-void * spine_mem_pool_tcm_mem_wait(int cpu_id) noexcept {
-    return spine_tcm_mem_try_wait(cpu_id, 1000 * 1000);
-}
-
-int spine_mem_pool_tcm_mem_release(int cpu_id) noexcept {
-    return spine_tcm_mem_release(cpu_id);
-}
-
-void * spine_mem_pool_alloc(size_t size, size_t alignment) noexcept {
-    try {
-        return get_spine_mem_pool_manager().alloc(size, alignment);
-    } catch (const std::bad_alloc &) {
-        GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: bad_alloc while allocating size %zu\n", __func__, size);
-        return nullptr;
-    }
-}
-
-void * spine_mem_pool_shared_mem_alloc(size_t size, size_t alignment) noexcept {
-    try {
-        return get_spine_mem_pool_shared_mem_manager().alloc(size, alignment);
-    } catch (const std::bad_alloc &) {
-        GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: bad_alloc while allocating shared memory size %zu\n", __func__, size);
-        return nullptr;
-    }
-}
-
-void spine_mem_pool_free(void * base) noexcept {
-    try {
-        get_spine_mem_pool_manager().free(base);
-    } catch (const std::bad_alloc &) {
-        GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: bad_alloc while freeing allocation %p\n", __func__, base);
-    }
-}
-
-void spine_mem_pool_shared_mem_free(void * base) noexcept {
-    try {
-        get_spine_mem_pool_shared_mem_manager().free(base);
-    } catch (const std::bad_alloc &) {
-        GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: bad_alloc while freeing shared allocation %p\n", __func__, base);
-    }
-}
-
-}  // namespace ggml::cpu::riscv64_spacemit
-
-extern "C" {
-void * ggml_backend_cpu_riscv64_spacemit_alloc_shared(size_t size, size_t alignment) {
-    void * result = ggml::cpu::riscv64_spacemit::spine_mem_pool_shared_mem_alloc(size, alignment);
-    if (result == nullptr) {
-        GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to allocate shared memory size %zu alignment %zu\n", __func__,
-                       size, alignment);
-    }
-    return result;
-}
-
-void ggml_backend_cpu_riscv64_spacemit_free_shared(void * ptr) {
-    ggml::cpu::riscv64_spacemit::spine_mem_pool_shared_mem_free(ptr);
-}
-}
@@ -1,32 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <cstdint>
-
-namespace ggml::cpu::riscv64_spacemit {
-
-enum class spine_mem_pool_backend : uint8_t {
-    none,
-    posix_memalign,
-    transparent_hugepage,
-    hugetlb_1g,
-};
-
-struct spine_mem_pool_tcm_info {
-    bool   available{ false };
-    size_t blk_size{ 0 };
-    size_t blk_num{ 0 };
-    bool   is_fake_tcm{ false };
-};
-
-bool   spine_mem_pool_tcm_init(spine_mem_pool_tcm_info * info) noexcept;
-void * spine_mem_pool_tcm_mem_get(int cpu_id) noexcept;
-void * spine_mem_pool_tcm_mem_wait(int cpu_id) noexcept;
-int    spine_mem_pool_tcm_mem_release(int cpu_id) noexcept;
-
-void * spine_mem_pool_alloc(size_t size, size_t alignment) noexcept;
-void * spine_mem_pool_shared_mem_alloc(size_t size, size_t alignment) noexcept;
-void   spine_mem_pool_free(void * base) noexcept;
-void   spine_mem_pool_shared_mem_free(void * base) noexcept;
-
-}  // namespace ggml::cpu::riscv64_spacemit
@@ -1,409 +0,0 @@
-#ifndef SPINE_TCM_PUBLIC_H_
-#define SPINE_TCM_PUBLIC_H_
-
-/*
- * spine_tcm public API
- *
- * Usage:
- *   1. Direct link mode
- *      Define SPINE_TCM_DIRECT_LINK and link against libspine_tcm.so.
- *
- *      if (spine_tcm_is_available()) {
- *          void *buffer = spine_tcm_mem_get(0);
- *          spine_tcm_mem_free(0);
- *      }
- *
- *   2. Header-only loader mode
- *      Include this header without linking libspine_tcm.so. The loader first
- *      tries to reuse a process-global spine_tcm instance and falls back to
- *      dlopen("libspine_tcm.so") when needed.
- *
- *      spine_tcm_open_handle(NULL);  // optional pre-bind
- *      if (spine_tcm_is_available()) {
- *          void *buffer = spine_tcm_mem_get(0);
- *          spine_tcm_mem_free(0);
- *      }
- */
-
-#include <stddef.h>
-#include <stdint.h>
-#include <string.h>
-
-#if !defined(SPINE_TCM_BUILD_SHARED) && !defined(SPINE_TCM_DIRECT_LINK)
-#    include <dlfcn.h>
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if defined(_WIN32)
-#    if defined(SPINE_TCM_BUILD_SHARED)
-#        define SPINE_TCM_API __declspec(dllexport)
-#    else
-#        define SPINE_TCM_API __declspec(dllimport)
-#    endif
-#else
-#    define SPINE_TCM_API __attribute__((visibility("default")))
-#endif
-
-typedef struct spine_tcm_mem_info {
-    size_t blk_size;
-    size_t blk_num;
-    int    is_fake_tcm;
-} spine_tcm_mem_info_t;
-
-typedef struct spine_tcm_block_info {
-    int      id;
-    void *   va;
-    size_t   size;
-    uint64_t phys_addr;
-    uint64_t cpu_affinity_mask;
-    int      owner_tid;
-    int      is_acquired;
-} spine_tcm_block_info_t;
-
-/* Shared-library runtime ABI exported by libspine_tcm.so. */
-SPINE_TCM_API const char * spine_tcm_runtime_version(void);
-SPINE_TCM_API int          spine_tcm_runtime_is_available(void);
-SPINE_TCM_API int          spine_tcm_runtime_layout_info(spine_tcm_mem_info_t * info);
-SPINE_TCM_API int          spine_tcm_runtime_mem_info(int id, spine_tcm_block_info_t * info);
-SPINE_TCM_API void *       spine_tcm_runtime_mem_get(int id);
-SPINE_TCM_API int          spine_tcm_runtime_mem_free(int id);
-SPINE_TCM_API void *       spine_tcm_runtime_mem_try_wait(int id, size_t timeout_us);
-SPINE_TCM_API int          spine_tcm_runtime_mem_release(int id);
-SPINE_TCM_API int          spine_tcm_runtime_mem_force_release(int id);
-SPINE_TCM_API int          spine_tcm_runtime_mem_query(int id);
-
-#if defined(SPINE_TCM_DIRECT_LINK)
-/* Optional no-op in direct-link mode. */
-static inline int spine_tcm_open_handle(const char * so_path) {
-    (void) so_path;
-    return 0;
-}
-
-static inline const char * spine_tcm_version(void) {
-    return spine_tcm_runtime_version();
-}
-
-/* Returns 1 when the runtime driver is available, otherwise 0. */
-static inline int spine_tcm_is_available(void) {
-    return spine_tcm_runtime_is_available();
-}
-
-/* Returns runtime memory geometry and whether the current backend is fake TCM. */
-static inline int spine_tcm_mem_info(spine_tcm_mem_info_t * info) {
-    return spine_tcm_runtime_layout_info(info);
-}
-
-/* Returns per-block runtime metadata for the given TCM id. */
-static inline int spine_tcm_block_info(int id, spine_tcm_block_info_t * info) {
-    return spine_tcm_runtime_mem_info(id, info);
-}
-
-/* Returns a cached buffer for the given TCM id, or NULL on failure. */
-static inline void * spine_tcm_mem_get(int id) {
-    return spine_tcm_runtime_mem_get(id);
-}
-
-/* Releases one reference acquired by spine_tcm_mem_get(id). */
-static inline int spine_tcm_mem_free(int id) {
-    return spine_tcm_runtime_mem_free(id);
-}
-
-/* Waits for a TCM block handoff and returns the driver-owned buffer when available. */
-static inline void * spine_tcm_mem_try_wait(int id, size_t over_time) {
-    return spine_tcm_runtime_mem_try_wait(id, over_time);
-}
-
-/* Releases a buffer acquired by spine_tcm_mem_try_wait(id, over_time). */
-static inline int spine_tcm_mem_release(int id) {
-    return spine_tcm_runtime_mem_release(id);
-}
-
-/* Forces a release for the given TCM id when the backend supports it. */
-static inline int spine_tcm_mem_force_release(int id) {
-    return spine_tcm_runtime_mem_force_release(id);
-}
-
-/* Returns whether the given TCM id is currently acquired. */
-static inline int spine_tcm_mem_query(int id) {
-    return spine_tcm_runtime_mem_query(id);
-}
-#elif !defined(SPINE_TCM_BUILD_SHARED)
-typedef struct spine_tcm_handle {
-    void * module_handle;
-    int    use_global_scope;
-    int    owns_module_handle;
-    const char * (*runtime_version)(void);
-    int (*runtime_is_available)(void);
-    int (*runtime_layout_info)(spine_tcm_mem_info_t * info);
-    int (*runtime_mem_info)(int id, spine_tcm_block_info_t * info);
-    void * (*runtime_mem_get)(int id);
-    int (*runtime_mem_free)(int id);
-    void * (*runtime_mem_try_wait)(int id, size_t over_time);
-    int (*runtime_mem_release)(int id);
-    int (*runtime_mem_force_release)(int id);
-    int (*runtime_mem_query)(int id);
-} spine_tcm_handle_t;
-
-static inline spine_tcm_handle_t * spine_tcm_default_handle(void) {
-    static spine_tcm_handle_t handle = { 0 };
-    return &handle;
-}
-
-static inline void spine_tcm_handle_reset(spine_tcm_handle_t * handle) {
-    if (handle != NULL) {
-        memset(handle, 0, sizeof(*handle));
-    }
-}
-
-static inline int spine_tcm_handle_bind(spine_tcm_handle_t * handle) {
-    void * symbol_scope = handle->use_global_scope ? RTLD_DEFAULT : handle->module_handle;
-
-    handle->runtime_version      = (const char * (*) (void) ) dlsym(symbol_scope, "spine_tcm_runtime_version");
-    handle->runtime_is_available = (int (*)(void)) dlsym(symbol_scope, "spine_tcm_runtime_is_available");
-    handle->runtime_layout_info =
-        (int (*)(spine_tcm_mem_info_t *)) dlsym(symbol_scope, "spine_tcm_runtime_layout_info");
-    handle->runtime_mem_info =
-        (int (*)(int, spine_tcm_block_info_t *)) dlsym(symbol_scope, "spine_tcm_runtime_mem_info");
-    handle->runtime_mem_get      = (void * (*) (int) ) dlsym(symbol_scope, "spine_tcm_runtime_mem_get");
-    handle->runtime_mem_free     = (int (*)(int)) dlsym(symbol_scope, "spine_tcm_runtime_mem_free");
-    handle->runtime_mem_try_wait = (void * (*) (int, size_t)) dlsym(symbol_scope, "spine_tcm_runtime_mem_try_wait");
-    handle->runtime_mem_release  = (int (*)(int)) dlsym(symbol_scope, "spine_tcm_runtime_mem_release");
-    handle->runtime_mem_force_release = (int (*)(int)) dlsym(symbol_scope, "spine_tcm_runtime_mem_force_release");
-    handle->runtime_mem_query         = (int (*)(int)) dlsym(symbol_scope, "spine_tcm_runtime_mem_query");
-
-    return handle->runtime_version != NULL && handle->runtime_is_available != NULL &&
-                   handle->runtime_layout_info != NULL && handle->runtime_mem_info != NULL &&
-                   handle->runtime_mem_get != NULL && handle->runtime_mem_free != NULL &&
-                   handle->runtime_mem_try_wait != NULL && handle->runtime_mem_release != NULL &&
-                   handle->runtime_mem_force_release != NULL && handle->runtime_mem_query != NULL ?
-               0 :
-               -1;
-}
-
-/*
- * Try to bind against an already-loaded process-global spine_tcm instance.
- * The shared library exports spine_tcm_runtime_marker only for this probe.
- */
-static inline int spine_tcm_try_bind_global(spine_tcm_handle_t * handle) {
-    if (dlsym(RTLD_DEFAULT, "spine_tcm_runtime_marker") == NULL) {
-        return -1;
-    }
-
-    handle->use_global_scope = 1;
-    return spine_tcm_handle_bind(handle);
-}
-
-/*
- * Optional pre-bind entry point.
- *
- * Behavior:
- *   - Reuses an already-loaded global spine_tcm instance when available.
- *   - Otherwise loads the shared library from so_path or the default soname.
- *   - Repeated calls are safe and return 0 after the first successful bind.
- */
-static inline int spine_tcm_open_handle(const char * so_path) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-    const char *         library  = (so_path != NULL && so_path[0] != '\0') ? so_path : "libspine_tcm.so";
-
-    if (resolved->module_handle != NULL || resolved->use_global_scope) {
-        return 0;
-    }
-
-    if (spine_tcm_try_bind_global(resolved) == 0) {
-        return 0;
-    }
-
-    spine_tcm_handle_reset(resolved);
-
-    resolved->module_handle      = dlopen(library, RTLD_LAZY | RTLD_GLOBAL);
-    resolved->owns_module_handle = resolved->module_handle != NULL ? 1 : 0;
-
-    if (resolved->module_handle == NULL) {
-        spine_tcm_handle_reset(resolved);
-        return -1;
-    }
-
-    if (spine_tcm_handle_bind(resolved) != 0) {
-        if (resolved->owns_module_handle) {
-            dlclose(resolved->module_handle);
-        }
-        spine_tcm_handle_reset(resolved);
-        return -1;
-    }
-
-    return 0;
-}
-
-/* Returns 1 when the runtime driver is available, otherwise 0. */
-static inline int spine_tcm_is_available(void) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_is_available == NULL) {
-        return 0;
-    }
-
-    return resolved->runtime_is_available();
-}
-
-/* Returns runtime memory geometry and whether the current backend is fake TCM. */
-static inline int spine_tcm_mem_info(spine_tcm_mem_info_t * info) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_layout_info == NULL) {
-        return -1;
-    }
-
-    return resolved->runtime_layout_info(info);
-}
-
-static inline const char * spine_tcm_version(void) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_version == NULL) {
-        return "unknown";
-    }
-
-    return resolved->runtime_version();
-}
-
-/* Returns per-block runtime metadata for the given TCM id. */
-static inline int spine_tcm_block_info(int id, spine_tcm_block_info_t * info) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_mem_info == NULL) {
-        return -1;
-    }
-
-    return resolved->runtime_mem_info(id, info);
-}
-
-/* Returns a cached buffer for the given TCM id, or NULL on failure. */
-static inline void * spine_tcm_mem_get(int id) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        return NULL;
-    }
-
-    if (resolved->runtime_mem_get == NULL) {
-        return NULL;
-    }
-
-    return resolved->runtime_mem_get(id);
-}
-
-/* Releases one reference acquired by spine_tcm_mem_get(id). */
-static inline int spine_tcm_mem_free(int id) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_mem_free == NULL) {
-        return -1;
-    }
-
-    return resolved->runtime_mem_free(id);
-}
-
-/* Waits for a TCM block handoff and returns the driver-owned buffer when available. */
-static inline void * spine_tcm_mem_try_wait(int id, size_t over_time) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        return NULL;
-    }
-
-    if (resolved->runtime_mem_try_wait == NULL) {
-        return NULL;
-    }
-
-    return resolved->runtime_mem_try_wait(id, over_time);
-}
-
-/* Releases a buffer acquired by spine_tcm_mem_try_wait(id, over_time). */
-static inline int spine_tcm_mem_release(int id) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_mem_release == NULL) {
-        return -1;
-    }
-
-    return resolved->runtime_mem_release(id);
-}
-
-/* Forces a release for the given TCM id when the backend supports it. */
-static inline int spine_tcm_mem_force_release(int id) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if ((resolved->module_handle == NULL && !resolved->use_global_scope) ||
-        resolved->runtime_mem_force_release == NULL) {
-        return -1;
-    }
-
-    return resolved->runtime_mem_force_release(id);
-}
-
-/* Returns whether the given TCM id is currently acquired. */
-static inline int spine_tcm_mem_query(int id) {
-    spine_tcm_handle_t * resolved = spine_tcm_default_handle();
-
-    if (resolved->module_handle == NULL && !resolved->use_global_scope) {
-        (void) spine_tcm_open_handle(NULL);
-    }
-
-    if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_mem_query == NULL) {
-        return -1;
-    }
-
-    return resolved->runtime_mem_query(id);
-}
-#else
-static inline const char * spine_tcm_version(void) {
-    return spine_tcm_runtime_version();
-}
-#endif
-
-#define SPINE_TCM_VERSION (spine_tcm_version())
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
@@ -1,971 +0,0 @@
-#include "allreduce.cuh"
-
-#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
-
-#include "convert.cuh"
-#include "ggml-impl.h"
-
-#include <algorithm>
-#include <cstdlib>
-#include <cstring>
-#include <limits>
-
-// ---------------------------------------------------------------------------
-// CUDA AllReduce for tensor-parallel inference across two GPUs.
-//
-// Provides an in-place sum reduction over matching tensors on two CUDA
-// devices in the same process.  Used by the tensor-split path alongside
-// NCCL; targets setups without NVLink, where data is exchanged between the
-// GPUs by staging it through pinned host memory over PCIe.
-//
-// Two reduction strategies are selected per call by tensor size:
-//
-//   * Chunked kernel path (small reductions): a single CUDA kernel both
-//     stages data through pinned host memory and performs the local sum.
-//     Cross-GPU synchronization happens *inside the kernel* (busy-wait on
-//     a host-memory flag), which keeps launch overhead low for the
-//     latency-sensitive token-generation case.
-//
-//   * Copy-engine path (large reductions): the transfer is split into
-//     D2H + H2D cudaMemcpyAsync chunks driven by the GPU's copy engine,
-//     followed by a small device-side add kernel.  Cross-GPU
-//     synchronization happens *outside the kernel*, via CUDA events
-//     between streams.  This keeps the compute engine free while large
-//     transfers are in flight, which matters for prefill-sized tensors.
-//     Reductions larger than the per-call inner cap are processed by an
-//     outer chunker that issues sequential inner calls.
-// ---------------------------------------------------------------------------
-
-// ---------------------------------------------------------------------------
-// Cross-GPU signal mechanism
-//
-// One int per (slot, rank) pair in pinned host memory.  Each AR call writes a
-// strictly increasing token (= the AR call number) into its own arrival int.
-// The peer spins until its read of the other's arrival int equals the token
-// it expects for this call -- a mismatch means the peer hasn't arrived yet.
-// Tokens never repeat over realistic call rates (32-bit int wraps in tens of
-// days at thousands of ARs/sec), so arrival ints don't need to be reset
-// between calls; we initialize once at pipeline init and let the values
-// accumulate.
-//
-// There is exactly one writer (the owning GPU) and one reader (the peer), so
-// we don't need atomics.  A volatile store paired with __threadfence_system()
-// provides the release ordering that makes the D2H writes visible system-wide
-// before the arrival token is observed.
-//
-// atomicAdd_system() requires hostNativeAtomicSupported, which is unavailable
-// on PCIe-attached consumer GPUs without NVLink, so the volatile path is the
-// portable choice.
-// ---------------------------------------------------------------------------
-
-static __device__ __forceinline__ void ggml_cuda_ar_signal_set(int * p, int token) {
-    *(volatile int *)p = token;
-}
-static __device__ __forceinline__ int ggml_cuda_ar_signal_get(const int * p) {
-    return *(const volatile int *)p;
-}
-
-// Byte spacing between adjacent arrival ints.  64 bytes (one cache line)
-// ensures each GPU/block's arrival slot lives on its own line, preventing
-// false-sharing stalls on the polling GPU.
-static constexpr size_t GGML_CUDA_AR_ARRIVAL_STRIDE = 64;
-
-// Number of blocks the chunked kernel launches with.  Each block stripes a
-// disjoint slice of the data and synchronizes through its own arrival-token
-// slot so multiple SMs can pump PCIe stores in parallel.
-static constexpr int GGML_CUDA_AR_KERNEL_BLOCKS = 8;
-
-// ---------------------------------------------------------------------------
-// Chunked kernel AllReduce -- 2 GPUs, supports float, half, and bfloat16.
-//
-// Both GPUs run this kernel simultaneously on independent streams.  sendbuf
-// and recvbuf live in T_dst (the caller's tensor type); host_mine / host_other
-// carry data in T_wire (the on-wire type, possibly narrower than T_dst -- e.g.
-// T_dst=F32 with T_wire=BF16 halves the bytes pushed across PCIe).  When
-// T_dst == T_wire the casts below are no-ops.
-//
-// Each GPU runs three phases:
-//
-//   Phase 1 (all threads): cast sendbuf (T_dst) -> T_wire and store as
-//                          single-instruction-width vectors into host_mine.
-//                          __threadfence_system() commits these writes to host
-//                          memory.
-//   Phase 2 (thread 0):    write token to arrival_mine; spin until
-//                          arrival_other == token.
-//   Phase 3 (all threads): read T_wire vectors from host_other, cast
-//                          each element to T_dst, and sum with the local
-//                          sendbuf value (also rounded through T_wire so that
-//                          both GPUs truncate identically -- this guarantees
-//                          bit-equivalent results across the two devices).
-//
-// Multi-block: blocks stripe vectors across (gridDim.x * blockDim.x) global
-// threads to keep multiple SMs issuing PCIe stores in parallel.  Each block
-// has its own arrival-token slot (offset by blockIdx.x * ARRIVAL_STRIDE);
-// thread 0 of each block signals/spins on that slot independently of other
-// blocks.  Tail elements (the leftover < ELEMS_PER_VEC at the end) are
-// handled only by block 0 to avoid cross-block writes to the same slots.
-// ---------------------------------------------------------------------------
-template <typename T_dst, typename T_wire>
-static __global__ void ggml_cuda_ar_kernel(
-        const T_dst  *              sendbuf,
-        T_dst        *              recvbuf,
-        T_wire       * __restrict__ host_mine,
-        const T_wire * __restrict__ host_other,
-        int                         count,
-        int *                       arrival_mine,
-        int *                       arrival_other,
-        int                         token) {
-
-    // Vector unit for the wire type, sized to the arch's widest single-instruction
-    // copy (16 B on Volta+).  Each phase-1 iter writes one vector to host memory;
-    // each phase-3 iter reads one and produces ELEMS_PER_VEC sums.
-    constexpr int ELEMS_PER_VEC = ggml_cuda_get_max_cpy_bytes() / sizeof(T_wire);
-    constexpr int ARRIVAL_INTS  = (int)(GGML_CUDA_AR_ARRIVAL_STRIDE / sizeof(int));
-
-    const int tid       = threadIdx.x;
-    const int nt        = blockDim.x;
-    const int bid       = blockIdx.x;
-    const int gtid      = bid * nt + tid;
-    const int gnt       = gridDim.x * nt;
-    const int count_vec = count / ELEMS_PER_VEC;
-    const int tail      = count_vec * ELEMS_PER_VEC;
-
-    // Phase 1: cast sendbuf (T_dst) -> host_mine (T_wire) and store as vectors.
-    {
-        for (int i = gtid; i < count_vec; i += gnt) {
-            const int off = i * ELEMS_PER_VEC;
-            T_wire wire[ELEMS_PER_VEC];
-            #pragma unroll
-            for (int k = 0; k < ELEMS_PER_VEC; ++k) {
-                wire[k] = ggml_cuda_cast<T_wire>(sendbuf[off + k]);
-            }
-            ggml_cuda_memcpy_1<sizeof(wire)>(&host_mine[off], wire);
-        }
-        if (bid == 0 && tid < count - tail) {
-            host_mine[tail + tid] = ggml_cuda_cast<T_wire>(sendbuf[tail + tid]);
-        }
-    }
-
-    // Commit this block's host writes before signalling.
-    __threadfence_system();
-    __syncthreads();
-
-    // Phase 2: thread 0 of each block signals on its own arrival slot, then
-    // spins for the matching slot from peer.  Per-block tokens mean blocks
-    // proceed independently -- no inter-block barrier needed.
-    if (tid == 0) {
-        int       * my_slot    = arrival_mine  + bid * ARRIVAL_INTS;
-        const int * other_slot = arrival_other + bid * ARRIVAL_INTS;
-
-        ggml_cuda_ar_signal_set(my_slot, token);
-        __threadfence_system(); // make our signal visible system-wide
-
-        while (ggml_cuda_ar_signal_get(other_slot) != token) {
-#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
-            __nanosleep(100);
-#else
-            NO_DEVICE_CODE;
-#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
-        }
-    }
-
-    __syncthreads();
-
-    // Acquire peer's host_other writes (this block's stripe of them).
-    __threadfence_system();
-
-    // Phase 3: read peer's T_wire vector, cast both sides through T_wire for
-    // bit-equivalence, sum in T_dst precision, and write back to recvbuf.
-    {
-        for (int i = gtid; i < count_vec; i += gnt) {
-            const int off = i * ELEMS_PER_VEC;
-            T_wire wire[ELEMS_PER_VEC];
-            ggml_cuda_memcpy_1<sizeof(wire)>(wire, &host_other[off]);
-            #pragma unroll
-            for (int k = 0; k < ELEMS_PER_VEC; ++k) {
-                const T_wire d_low = ggml_cuda_cast<T_wire>(sendbuf[off + k]);
-                recvbuf[off + k] = ggml_cuda_cast<T_dst>(
-                    ggml_cuda_cast<float>(d_low) + ggml_cuda_cast<float>(wire[k]));
-            }
-        }
-        if (bid == 0 && tid < count - tail) {
-            const T_wire d_low = ggml_cuda_cast<T_wire>(sendbuf[tail + tid]);
-            recvbuf[tail + tid] = ggml_cuda_cast<T_dst>(
-                ggml_cuda_cast<float>(d_low) +
-                ggml_cuda_cast<float>(host_other[tail + tid]));
-        }
-    }
-}
-
-// Combined load-convert-add kernel.  The peer's contribution arrives as T_src
-// (which may be a lower-precision type than T_dst when the BF16 round-trip is
-// active).  For bit-equivalence between the two GPUs, dst is first rounded
-// through T_src's precision via ggml_cuda_cast -- peer already truncated its
-// own value the same way before sending -- so both sides perform identical
-// arithmetic.  When T_dst == T_src the round-trip cast is a no-op.
-template <typename T_dst, typename T_src>
-static __global__ void ggml_cuda_ar_add_kernel(
-        T_dst       * __restrict__ dst,
-        const T_src * __restrict__ src,
-        int count) {
-    const int tid = blockIdx.x * blockDim.x + threadIdx.x;
-    const int nt  = gridDim.x * blockDim.x;
-    for (int i = tid; i < count; i += nt) {
-        const T_src d_low = ggml_cuda_cast<T_src>(dst[i]);
-        dst[i] = ggml_cuda_cast<T_dst>(
-            ggml_cuda_cast<float>(d_low) + ggml_cuda_cast<float>(src[i]));
-    }
-}
-
-// ---------------------------------------------------------------------------
-// Pipeline structure
-// ---------------------------------------------------------------------------
-
-// Number of slots in the event / arrival ring.  Two slots is sufficient:
-// lockstep guarantees the two GPUs are at most one AR (or chunk) apart, so
-// slot[N%2] is always safe to reuse -- peer has already consumed slot[N%2]
-// from AR N-2 by the time we get to AR N.  acquire_slot's
-// cudaEventSynchronize on ev.ker for both devices makes that consumption
-// explicit before we overwrite host_buf[slot] for the new AR.
-static constexpr int GGML_CUDA_AR_POOL_SIZE = 2;
-
-// Maximum chunk size (bytes per GPU) handled by one chunked kernel launch.
-// Larger tensors are reduced by issuing multiple chunked launches.
-static constexpr size_t GGML_CUDA_AR_MAX_BYTES = 1024 * 1024; // 1 MB
-
-// Copy-engine path: largest tensor accepted on this path; sets host_large /
-// dev_tmp allocation size.
-static constexpr size_t GGML_CUDA_AR_COPY_MAX_BYTES = 32 * 1024 * 1024; // 32 MB
-
-// AR wire size at which the copy-engine path takes over from the chunked-
-// kernel path.  Override via GGML_CUDA_AR_COPY_THRESHOLD.
-static constexpr size_t GGML_CUDA_AR_COPY_THRESHOLD_DEFAULT = 1024 * 1024; // 1 MB
-// Per-call CE chunk-size heuristic: chunk_bytes = clamp(nbytes / 4, MIN, MAX).
-// The /4 keeps ~4 chunks in flight at any moment (good D2H/H2D overlap with
-// the peer); the clamps cover the cases where nbytes/4 is too small (per-
-// memcpy fixed cost dominates) or too large (chunk-level pipelining stalls).
-// Env var GGML_CUDA_AR_COPY_CHUNK_BYTES can override with a fixed value.
-static constexpr size_t GGML_CUDA_AR_COPY_CHUNK_BYTES_HEURISTIC_MIN = 512 * 1024;       // 512 KB
-static constexpr size_t GGML_CUDA_AR_COPY_CHUNK_BYTES_HEURISTIC_MAX = 2 * 1024 * 1024;  // 2 MB
-// Absolute floor that an env-var override is allowed to set; this caps the
-// per-slot copy-event array.  256 KB -> up to 128 chunks per 32 MB tensor.
-static constexpr size_t GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN = 256 * 1024;
-static constexpr int GGML_CUDA_AR_COPY_MAX_CHUNKS =
-    static_cast<int>((GGML_CUDA_AR_COPY_MAX_BYTES + GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN - 1) /
-                    GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN);
-
-struct ggml_cuda_ar_event_slot {
-    cudaEvent_t app = nullptr;  // upstream computation complete
-    cudaEvent_t cpy[GGML_CUDA_AR_COPY_MAX_CHUNKS] = {};  // copy-engine D2H chunks complete
-    cudaEvent_t h2d = nullptr;  // copy-engine H2Ds complete (handoff AR stream -> compute stream)
-    cudaEvent_t ker = nullptr;  // AllReduce kernel complete
-};
-
-// Mapped pinned host allocation: cudaHostAlloc + cudaHostGetDevicePointer
-// in one place, with the host handle preserved for cudaFreeHost.  Used where
-// the CPU never touches the buffer -- only the device reads/writes via the
-// mapped device pointer.  Required on systems where cudaDevAttrCanUseHost-
-// PointerForRegisteredMem is 0 and the host pointer can't be used as a
-// device pointer.
-struct ggml_cuda_ar_host_mapping {
-    uint8_t * host = nullptr;   // cudaFreeHost handle; also the H-side ptr for cudaMemcpyAsync
-    uint8_t * dev  = nullptr;   // device-side pointer for kernels / cudaMemset
-
-    cudaError_t alloc(size_t bytes) {
-        cudaError_t rc = cudaHostAlloc(reinterpret_cast<void **>(&host), bytes,
-                                       cudaHostAllocPortable | cudaHostAllocMapped);
-        if (rc != cudaSuccess) {
-            host = nullptr;
-            return rc;
-        }
-        rc = cudaHostGetDevicePointer(reinterpret_cast<void **>(&dev), host, 0);
-        if (rc != cudaSuccess) {
-            cudaFreeHost(host);
-            host = nullptr;
-            dev  = nullptr;
-        }
-        return rc;
-    }
-
-    void free() {
-        if (host) {
-            cudaFreeHost(host);
-            host = nullptr;
-            dev  = nullptr;
-        }
-    }
-};
-
-struct ggml_cuda_ar_pipeline {
-    int      n_devices;
-    int      devices[GGML_CUDA_MAX_DEVICES];
-    size_t   buf_bytes;    // bytes per device in host_buf[]
-    size_t   copy_bytes;   // bytes per device in host_large[] / dev_tmp[]
-    size_t   copy_threshold;
-    size_t   copy_chunk_bytes;
-    size_t   bf16_threshold; // tensors >= this size (bytes) are reduced via FP32->BF16 round-trip; 0 disables
-    uint64_t call_count;
-
-    // Per-device resources.
-    ggml_cuda_ar_host_mapping host_buf[GGML_CUDA_MAX_DEVICES];   // pinned staging (chunked kernel)
-    ggml_cuda_ar_host_mapping host_large[GGML_CUDA_MAX_DEVICES]; // pinned staging (copy-engine)
-    char *                    dev_tmp[GGML_CUDA_MAX_DEVICES];    // device scratch for copy-engine path
-    cudaStream_t             streams[GGML_CUDA_MAX_DEVICES];   // non-blocking
-    ggml_cuda_ar_event_slot  ev_pool[GGML_CUDA_MAX_DEVICES][GGML_CUDA_AR_POOL_SIZE];
-
-    // Copy-engine: per-device "I finished reading my peer's host_large"
-    // event.  Indexed by RECORDER device.  Recorded same-device on streams[i]
-    // after stage 2's last H2D from host_large[peer].  Waited cross-device
-    // by peer's stage-1 stream before the next AR overwrites host_large[peer].
-    cudaEvent_t              host_large_read_done[GGML_CUDA_MAX_DEVICES];
-    bool                     host_large_read_done_valid;
-
-    // Copy-engine: per-device "my add_kernel is done with dev_tmp" event.
-    // Recorded on the compute stream after each add_kernel; the AR stream
-    // waits on it before the next copy_impl's H2D overwrites dev_tmp.  Lets us
-    // single-buffer dev_tmp despite add_kernel running on a separate stream.
-    cudaEvent_t              dev_tmp_kernel_done[GGML_CUDA_MAX_DEVICES];
-    bool                     dev_tmp_kernel_done_valid;
-
-    // Arrival ring: ARRIVAL_STRIDE bytes between adjacent ints.  Mapped pinned
-    // memory; CPU never reads/writes -- only the kernel and cudaMemset.
-    // Use ggml_cuda_ar_arrival_ptr() to index.
-    ggml_cuda_ar_host_mapping arrival;
-};
-
-// Base pointer for the (slot, rank) per-block token block.  The kernel adds
-// blockIdx.x * (ARRIVAL_STRIDE/sizeof(int)) internally to land on its own slot.
-static int * ggml_cuda_ar_arrival_ptr(const ggml_cuda_ar_pipeline * p, int slot, int rank) {
-    const size_t offset = ((size_t)slot * p->n_devices + rank) *
-                          GGML_CUDA_AR_KERNEL_BLOCKS * GGML_CUDA_AR_ARRIVAL_STRIDE;
-    return reinterpret_cast<int *>(p->arrival.dev + offset);
-}
-
-static uint64_t ggml_cuda_ar_env_u64(const char * name, uint64_t default_value) {
-    const char * value = getenv(name);
-    if (value == nullptr || value[0] == '\0') {
-        return default_value;
-    }
-
-    char * end = nullptr;
-    const unsigned long long parsed = strtoull(value, &end, 10);
-    return end != value ? (uint64_t) parsed : default_value;
-}
-
-struct ggml_cuda_ar_slot_info {
-    int slot;
-    int token;
-};
-
-static ggml_cuda_ar_slot_info ggml_cuda_ar_acquire_slot(ggml_cuda_ar_pipeline * p) {
-    const int  slot        = static_cast<int>(p->call_count % GGML_CUDA_AR_POOL_SIZE);
-    const bool pool_lapped = p->call_count >= GGML_CUDA_AR_POOL_SIZE;
-    p->call_count++;
-
-    if (pool_lapped) {
-        for (int i = 0; i < p->n_devices; ++i) {
-            ggml_cuda_set_device(p->devices[i]);
-            CUDA_CHECK(cudaEventSynchronize(p->ev_pool[i][slot].ker));
-        }
-    }
-
-    return { slot, (int) p->call_count };
-}
-
-// Per-AR copy-engine chunk size: env-var override if set, else heuristic
-// (clamp(nbytes/4, HEURISTIC_MIN, HEURISTIC_MAX)).
-static size_t ggml_cuda_ar_chunk_bytes(const ggml_cuda_ar_pipeline * p, size_t nbytes) {
-    if (p->copy_chunk_bytes > 0) {
-        return p->copy_chunk_bytes;
-    }
-    return std::min(GGML_CUDA_AR_COPY_CHUNK_BYTES_HEURISTIC_MAX,
-                    std::max(GGML_CUDA_AR_COPY_CHUNK_BYTES_HEURISTIC_MIN, nbytes / 4));
-}
-
-static void ggml_cuda_ar_wait_for_compute(
-        ggml_cuda_ar_pipeline * p, ggml_backend_cuda_context * cuda_ctx, int rank, int slot) {
-    ggml_cuda_ar_event_slot & ev = p->ev_pool[rank][slot];
-    CUDA_CHECK(cudaEventRecord(ev.app, cuda_ctx->stream()));
-    CUDA_CHECK(cudaStreamWaitEvent(p->streams[rank], ev.app));
-}
-
-// ---------------------------------------------------------------------------
-// Init / free
-// ---------------------------------------------------------------------------
-
-ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(const int * devices, size_t n_devices) {
-
-    if (n_devices != 2) {
-        GGML_LOG_DEBUG("%s: internal AllReduce only supports n_devices=2 (got %zu); "
-                       "falling back\n", __func__, n_devices);
-        return nullptr;
-    }
-
-    // The chunked kernel uses __nanosleep, which is sm70+ (Volta+).
-    for (size_t i = 0; i < n_devices; ++i) {
-        const int cc = ggml_cuda_info().devices[devices[i]].cc;
-        if (cc < GGML_CUDA_CC_VOLTA) {
-            GGML_LOG_DEBUG("%s: internal AllReduce requires compute capability >= %d "
-                           "(device %d has cc=%d); falling back\n",
-                           __func__, GGML_CUDA_CC_VOLTA, devices[i], cc);
-            return nullptr;
-        }
-    }
-
-    auto * p = new ggml_cuda_ar_pipeline{};
-    p->n_devices        = n_devices;
-    p->copy_bytes       = GGML_CUDA_AR_COPY_MAX_BYTES;
-    p->copy_threshold   = ggml_cuda_ar_env_u64("GGML_CUDA_AR_COPY_THRESHOLD", GGML_CUDA_AR_COPY_THRESHOLD_DEFAULT);
-    // 0 = use the per-call heuristic (default).  Non-zero env value forces a
-    // fixed chunk size for diagnostics, with a floor at COPY_CHUNK_BYTES_MIN.
-    p->copy_chunk_bytes = ggml_cuda_ar_env_u64("GGML_CUDA_AR_COPY_CHUNK_BYTES", 0);
-    if (p->copy_chunk_bytes > 0 && p->copy_chunk_bytes < GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN) {
-        GGML_LOG_WARN("%s: GGML_CUDA_AR_COPY_CHUNK_BYTES=%zu below minimum %zu; clamping\n",
-                      __func__, p->copy_chunk_bytes, GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN);
-        p->copy_chunk_bytes = GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN;
-    }
-    // Default 1: BF16 round-trip is always on for F32 inputs (any non-zero
-    // ne).  Set GGML_CUDA_AR_BF16_THRESHOLD=0 to disable, or to a larger
-    // byte threshold to opt out for small tensors.
-    p->bf16_threshold   = ggml_cuda_ar_env_u64("GGML_CUDA_AR_BF16_THRESHOLD", 1);
-    for (size_t i = 0; i < n_devices; ++i) {
-        p->devices[i] = devices[i];
-    }
-
-    // Per-device streams and event pools.
-    for (size_t i = 0; i < n_devices; ++i) {
-        ggml_cuda_set_device(p->devices[i]);
-
-        cudaStream_t stream = nullptr;
-        if (cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking) != cudaSuccess) {
-            GGML_LOG_ERROR("%s: cudaStreamCreateWithFlags failed for device %d\n",
-                           __func__, p->devices[i]);
-            ggml_cuda_ar_pipeline_free(p);
-            return nullptr;
-        }
-        p->streams[i] = stream;
-
-        for (int s = 0; s < GGML_CUDA_AR_POOL_SIZE; ++s) {
-            bool ok =
-                cudaEventCreateWithFlags(&p->ev_pool[i][s].app, cudaEventDisableTiming) == cudaSuccess &&
-                cudaEventCreateWithFlags(&p->ev_pool[i][s].h2d, cudaEventDisableTiming) == cudaSuccess &&
-                cudaEventCreateWithFlags(&p->ev_pool[i][s].ker, cudaEventDisableTiming) == cudaSuccess;
-            for (int c = 0; ok && c < GGML_CUDA_AR_COPY_MAX_CHUNKS; ++c) {
-                ok = cudaEventCreateWithFlags(&p->ev_pool[i][s].cpy[c], cudaEventDisableTiming) == cudaSuccess;
-            }
-            if (!ok) {
-                GGML_LOG_ERROR("%s: cudaEventCreate failed for device %d slot %d\n",
-                               __func__, p->devices[i], s);
-                ggml_cuda_ar_pipeline_free(p);
-                return nullptr;
-            }
-        }
-
-        if (cudaEventCreateWithFlags(&p->host_large_read_done[i], cudaEventDisableTiming) != cudaSuccess) {
-            GGML_LOG_ERROR("%s: cudaEventCreate for host_large_read_done failed for device %d\n",
-                           __func__, p->devices[i]);
-            ggml_cuda_ar_pipeline_free(p);
-            return nullptr;
-        }
-        if (cudaEventCreateWithFlags(&p->dev_tmp_kernel_done[i], cudaEventDisableTiming) != cudaSuccess) {
-            GGML_LOG_ERROR("%s: cudaEventCreate for dev_tmp_kernel_done failed for device %d\n",
-                           __func__, p->devices[i]);
-            ggml_cuda_ar_pipeline_free(p);
-            return nullptr;
-        }
-    }
-
-    // Arrival ring: cache-line padded so each GPU's int is on its own line.
-    const size_t arrival_bytes =
-        (size_t)GGML_CUDA_AR_POOL_SIZE * n_devices *
-        GGML_CUDA_AR_KERNEL_BLOCKS * GGML_CUDA_AR_ARRIVAL_STRIDE;
-    if (p->arrival.alloc(arrival_bytes) != cudaSuccess) {
-        GGML_LOG_ERROR("%s: alloc for arrival ring failed (%zu bytes)\n",
-                       __func__, arrival_bytes);
-        ggml_cuda_ar_pipeline_free(p);
-        return nullptr;
-    }
-    ggml_cuda_set_device(p->devices[0]);
-    if (cudaMemset(p->arrival.dev, 0, arrival_bytes) != cudaSuccess) {
-        GGML_LOG_ERROR("%s: cudaMemset for arrival ring failed (%zu bytes)\n",
-                       __func__, arrival_bytes);
-        ggml_cuda_ar_pipeline_free(p);
-        return nullptr;
-    }
-
-    // Per-device pinned staging buffers -- POOL_SIZE-deep ring so the chunked-
-    // kernel can write the next slot's data while the peer is still reading
-    // the previous slot's. Indexed by (slot * buf_bytes) at the call site.
-    p->buf_bytes = GGML_CUDA_AR_MAX_BYTES;
-    const size_t host_buf_total = (size_t) GGML_CUDA_AR_POOL_SIZE * p->buf_bytes;
-    for (size_t i = 0; i < n_devices; ++i) {
-        if (p->host_buf[i].alloc(host_buf_total) != cudaSuccess) {
-            GGML_LOG_ERROR("%s: alloc for staging failed (%zu bytes)\n",
-                           __func__, host_buf_total);
-            ggml_cuda_ar_pipeline_free(p);
-            return nullptr;
-        }
-    }
-
-    // Copy-engine path: pinned host staging + device scratch, sized for the
-    // largest tensor we accept on this path (GGML_CUDA_AR_COPY_MAX_BYTES).
-    // dev_tmp is single-buffered; cross-AR safety is enforced by an explicit
-    // cross-stream wait in copy_impl on the prior AR's add_kernel-done event.
-    for (size_t i = 0; i < n_devices; ++i) {
-        ggml_cuda_set_device(p->devices[i]);
-        if (p->host_large[i].alloc(p->copy_bytes) != cudaSuccess) {
-            GGML_LOG_ERROR("%s: alloc for large staging failed (%zu bytes)\n",
-                           __func__, p->copy_bytes);
-            ggml_cuda_ar_pipeline_free(p);
-            return nullptr;
-        }
-        if (cudaMalloc(reinterpret_cast<void **>(&p->dev_tmp[i]), p->copy_bytes) != cudaSuccess) {
-            GGML_LOG_ERROR("%s: cudaMalloc for copy scratch failed (%zu bytes) on device %d\n",
-                           __func__, p->copy_bytes, p->devices[i]);
-            ggml_cuda_ar_pipeline_free(p);
-            return nullptr;
-        }
-    }
-
-    GGML_LOG_INFO("%s: initialized AllReduce pipeline: %zu GPUs, "
-                  "%zu KB chunked kernel staging + %zu MB copy-engine staging per GPU\n",
-                  __func__, n_devices, p->buf_bytes >> 10, p->copy_bytes >> 20);
-
-    return p;
-}
-
-void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * p) {
-    if (!p) {
-        return;
-    }
-
-    // Drain all in-flight kernels before tearing down resources.
-    for (int i = 0; i < p->n_devices; ++i) {
-        if (p->streams[i]) {
-            ggml_cuda_set_device(p->devices[i]);
-            cudaStreamSynchronize(p->streams[i]);
-        }
-    }
-
-    for (int i = 0; i < p->n_devices; ++i) {
-        p->host_buf[i].free();
-        p->host_large[i].free();
-        if (p->dev_tmp[i]) {
-            ggml_cuda_set_device(p->devices[i]);
-            cudaFree(p->dev_tmp[i]);
-        }
-        ggml_cuda_set_device(p->devices[i]);
-        for (int s = 0; s < GGML_CUDA_AR_POOL_SIZE; ++s) {
-            if (p->ev_pool[i][s].app) { cudaEventDestroy(p->ev_pool[i][s].app); }
-            for (int c = 0; c < GGML_CUDA_AR_COPY_MAX_CHUNKS; ++c) {
-                if (p->ev_pool[i][s].cpy[c]) { cudaEventDestroy(p->ev_pool[i][s].cpy[c]); }
-            }
-            if (p->ev_pool[i][s].h2d) { cudaEventDestroy(p->ev_pool[i][s].h2d); }
-            if (p->ev_pool[i][s].ker) { cudaEventDestroy(p->ev_pool[i][s].ker); }
-        }
-        if (p->host_large_read_done[i]) {
-            ggml_cuda_set_device(p->devices[i]);
-            cudaEventDestroy(p->host_large_read_done[i]);
-        }
-        if (p->dev_tmp_kernel_done[i]) {
-            ggml_cuda_set_device(p->devices[i]);
-            cudaEventDestroy(p->dev_tmp_kernel_done[i]);
-        }
-        if (p->streams[i]) {
-            ggml_cuda_set_device(p->devices[i]);
-            cudaStreamDestroy(p->streams[i]);
-        }
-    }
-    p->arrival.free();
-    delete p;
-}
-
-// ---------------------------------------------------------------------------
-// Dispatch
-// ---------------------------------------------------------------------------
-
-// Asymmetric copy_impl: data sent over PCIe in T_src precision (one element of
-// nbytes per ne element); accumulated locally into a T_dst buffer.  When
-// T_src == T_dst this is the original homogeneous reduction.  When they differ
-// (e.g. BF16 wire / F32 accumulator) the add kernel rounds dst through T_src
-// for bit-equivalence between GPUs and we skip the otherwise-needed
-// post-conversion entirely.
-template <typename T_src, typename T_dst>
-static bool ggml_cuda_ar_allreduce_copy_impl(
-        ggml_cuda_ar_pipeline * p,
-        ggml_backend_t        * backends,
-        T_src * const           src_buf[GGML_CUDA_MAX_DEVICES],
-        T_dst * const           dst_buf[GGML_CUDA_MAX_DEVICES],
-        const bool              compute[GGML_CUDA_MAX_DEVICES],
-        int64_t                 ne,
-        size_t                  nbytes) {
-    GGML_ASSERT(p->n_devices == 2);
-    GGML_ASSERT(nbytes <= p->copy_bytes);
-    GGML_ASSERT(ne <= std::numeric_limits<int>::max());
-
-    const size_t chunk_bytes = ggml_cuda_ar_chunk_bytes(p, nbytes);
-    GGML_ASSERT(chunk_bytes > 0);
-
-    const int slot = ggml_cuda_ar_acquire_slot(p).slot;
-    const size_t copy_chunks = (nbytes + chunk_bytes - 1) / chunk_bytes;
-    GGML_ASSERT(copy_chunks <= GGML_CUDA_AR_COPY_MAX_CHUNKS);
-
-    ggml_backend_cuda_context * cuda_ctx[2] = {};
-
-    // Stage 1: both GPUs copy their local contribution to pinned host memory.
-    for (int i = 0; i < 2; ++i) {
-        ggml_cuda_set_device(p->devices[i]);
-        cuda_ctx[i] = static_cast<ggml_backend_cuda_context *>(backends[i]->context);
-        GGML_ASSERT(cuda_ctx[i]->device == p->devices[i]);
-
-        ggml_cuda_ar_wait_for_compute(p, cuda_ctx[i], i, slot);
-
-        // Wait for peer's H2D from our host_large[i] (recorded in the
-        // previous AR's stage 2) to complete before we overwrite host_large[i].
-        // host_large_read_done[peer] = peer finished reading host_large[i].
-        // No-op on the first AR -- no prior record exists.
-        if (p->host_large_read_done_valid) {
-            const int peer = 1 - i;
-            CUDA_CHECK(cudaStreamWaitEvent(p->streams[i], p->host_large_read_done[peer]));
-        }
-
-        if (!compute[i]) {
-            CUDA_CHECK(cudaMemsetAsync(src_buf[i], 0, nbytes, p->streams[i]));
-        }
-
-        for (size_t c = 0; c < copy_chunks; ++c) {
-            const size_t offset = c * chunk_bytes;
-            const size_t this_bytes = (nbytes - offset) < chunk_bytes ?
-                (nbytes - offset) : chunk_bytes;
-
-            CUDA_CHECK(cudaMemcpyAsync(
-                p->host_large[i].host + offset, reinterpret_cast<char *>(src_buf[i]) + offset, this_bytes,
-                cudaMemcpyDeviceToHost, p->streams[i]));
-            CUDA_CHECK(cudaEventRecord(p->ev_pool[i][slot].cpy[c], p->streams[i]));
-        }
-    }
-
-    // Stage 2: each GPU waits for each peer D2H chunk, pulls that chunk back to
-    // local device scratch (dev_tmp), then performs one device-local add over
-    // the assembled peer tensor.  The H2Ds run on the AR stream (copy engine)
-    // and the add_kernel runs on the caller's compute stream, so the AR stream
-    // stays pure-copy and avoids an in-stream copy->compute engine switch every
-    // AR.  dev_tmp is single-buffered: the AR stream waits cross-stream on the
-    // prior AR's add_kernel-done event before overwriting it.
-    for (int i = 0; i < 2; ++i) {
-        const int peer = 1 - i;
-        ggml_cuda_set_device(p->devices[i]);
-
-        // Wait for the previous AR's add_kernel (on the compute stream) to
-        // finish reading dev_tmp before our H2D overwrites it.  No-op on the
-        // first copy_impl call.
-        if (p->dev_tmp_kernel_done_valid) {
-            CUDA_CHECK(cudaStreamWaitEvent(p->streams[i], p->dev_tmp_kernel_done[i]));
-        }
-
-        for (size_t c = 0; c < copy_chunks; ++c) {
-            const size_t offset = c * chunk_bytes;
-            const size_t this_bytes = (nbytes - offset) < chunk_bytes ?
-                (nbytes - offset) : chunk_bytes;
-
-            CUDA_CHECK(cudaStreamWaitEvent(p->streams[i], p->ev_pool[peer][slot].cpy[c]));
-            CUDA_CHECK(cudaMemcpyAsync(
-                p->dev_tmp[i] + offset, p->host_large[peer].host + offset, this_bytes,
-                cudaMemcpyHostToDevice, p->streams[i]));
-        }
-
-        // Mark our reads of host_large[peer] complete so peer's next AR can
-        // safely overwrite it.
-        CUDA_CHECK(cudaEventRecord(p->host_large_read_done[i], p->streams[i]));
-
-        // Hand off from AR stream (copy engine) to compute stream: compute
-        // stream waits for all H2Ds to finish, then runs the add_kernel.
-        CUDA_CHECK(cudaEventRecord(p->ev_pool[i][slot].h2d, p->streams[i]));
-        CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx[i]->stream(), p->ev_pool[i][slot].h2d));
-
-        const int block_size = 256;
-        int n_blocks = (int) ((ne + block_size - 1) / block_size);
-        if (n_blocks > 1024) {
-            n_blocks = 1024;
-        }
-        ggml_cuda_ar_add_kernel<T_dst, T_src><<<n_blocks, block_size, 0, cuda_ctx[i]->stream()>>>(
-            dst_buf[i],
-            reinterpret_cast<const T_src *>(p->dev_tmp[i]),
-            (int) ne);
-        CUDA_CHECK(cudaGetLastError());
-
-        // Record dev_tmp-released on the compute stream so the next copy_impl
-        // can wait for the kernel to finish before overwriting dev_tmp.  Also
-        // record AR-done as ev.ker for acquire_slot's pool-wraparound sync.
-        CUDA_CHECK(cudaEventRecord(p->dev_tmp_kernel_done[i], cuda_ctx[i]->stream()));
-        CUDA_CHECK(cudaEventRecord(p->ev_pool[i][slot].ker, cuda_ctx[i]->stream()));
-    }
-    p->host_large_read_done_valid = true;
-    p->dev_tmp_kernel_done_valid = true;
-
-    return true;
-}
-
-// Outer-level chunker: copy_impl handles up to copy_bytes per call (limited by
-// the host_large / dev_tmp allocation size).  When the full AR exceeds that,
-// slice the tensor into copy_bytes-sized pieces and call copy_impl repeatedly.
-// Each slice goes through its own stage 1 -> stage 2 cycle and acquires its own
-// slot, so cross-AR fences and pool wraparound work the same way as for any
-// other sequence of small ARs.
-template <typename T_src, typename T_dst>
-static bool ggml_cuda_ar_allreduce_copy_outer(
-        ggml_cuda_ar_pipeline * p,
-        ggml_backend_t        * backends,
-        T_src * const           src_buf[GGML_CUDA_MAX_DEVICES],
-        T_dst * const           dst_buf[GGML_CUDA_MAX_DEVICES],
-        const bool              compute[GGML_CUDA_MAX_DEVICES],
-        int64_t                 ne) {
-    const int64_t outer_max_elems = (int64_t) (p->copy_bytes / sizeof(T_src));
-    GGML_ASSERT(outer_max_elems > 0);
-
-    bool ok = true;
-    for (int64_t outer_start = 0; outer_start < ne && ok; outer_start += outer_max_elems) {
-        const int64_t outer_ne     = std::min(outer_max_elems, ne - outer_start);
-        const size_t  outer_nbytes = (size_t) outer_ne * sizeof(T_src);
-
-        T_src * src[GGML_CUDA_MAX_DEVICES] = {};
-        T_dst * dst[GGML_CUDA_MAX_DEVICES] = {};
-        for (int i = 0; i < p->n_devices; ++i) {
-            src[i] = src_buf[i] + outer_start;
-            dst[i] = dst_buf[i] + outer_start;
-        }
-        ok = ggml_cuda_ar_allreduce_copy_impl<T_src, T_dst>(
-            p, backends, src, dst, compute, outer_ne, outer_nbytes);
-    }
-    return ok;
-}
-
-bool ggml_cuda_ar_allreduce(
-        ggml_cuda_ar_pipeline * p,
-        ggml_backend_t        * backends,
-        ggml_tensor           ** tensors) {
-    GGML_ASSERT(p != nullptr);
-
-    const int n = p->n_devices;
-    GGML_ASSERT(n == 2);
-
-    const ggml_type input_type = tensors[0]->type;
-    GGML_ASSERT(input_type == GGML_TYPE_F32 || input_type == GGML_TYPE_F16 || input_type == GGML_TYPE_BF16);
-
-    const int64_t ne = ggml_nelements(tensors[0]);
-    GGML_ASSERT(ne > 0);
-
-    const size_t   input_nbytes = ggml_nbytes(tensors[0]);
-
-    // BF16 round-trip: F32 inputs >= bf16_threshold are converted to BF16 for
-    // the reduction (chunked or copy-engine), halving on-wire bytes. Matches
-    // NCCL's behaviour. The pre-conversion zeroes inactive shards so the
-    // inner paths see them as already-prepared compute tensors.
-    const bool use_bf16 =
-        input_type == GGML_TYPE_F32 &&
-        p->bf16_threshold > 0 &&
-        input_nbytes >= p->bf16_threshold;
-
-    const ggml_type kernel_type = use_bf16 ? GGML_TYPE_BF16 : input_type;
-    const size_t    type_size   = ggml_type_size(kernel_type);
-    GGML_ASSERT(p->buf_bytes >= type_size);
-    const size_t    nbytes      = (size_t) ne * type_size;
-
-    bool compute_flag[GGML_CUDA_MAX_DEVICES] = {};
-    for (int i = 0; i < n; ++i) {
-        compute_flag[i] = (tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) != 0;
-    }
-
-    // Decide between copy-engine and chunked kernel paths based on the working
-    // type's actual byte count.  No upper bound: copy_outer slices reductions
-    // larger than copy_bytes into copy_bytes-sized pieces.
-    const bool use_copy_engine =
-        p->copy_threshold > 0 &&
-        nbytes >= p->copy_threshold;
-
-    // BF16 inactive-shard zeroing: when use_bf16 is on, the combined kernel
-    // (chunked kernel path) and the combined add kernel (copy_engine path)
-    // both accumulate into the F32 tensor data directly, so an inactive
-    // shard's accumulator must start at zero.
-    if (use_bf16) {
-        for (int i = 0; i < n; ++i) {
-            if (!compute_flag[i]) {
-                auto * cuda_ctx = static_cast<ggml_backend_cuda_context *>(backends[i]->context);
-                GGML_ASSERT(cuda_ctx->device == p->devices[i]);
-                ggml_cuda_set_device(p->devices[i]);
-                CUDA_CHECK(cudaMemsetAsync(tensors[i]->data, 0, (size_t) ne * sizeof(float), cuda_ctx->stream()));
-            }
-        }
-    }
-
-    // Pre-convert F32 -> BF16 into bf16_tmp ONLY for the copy_engine + use_bf16
-    // path; the chunked kernel path's combined kernel does the conversion
-    // inline as it writes to host_buf.
-    ggml_cuda_pool_alloc<nv_bfloat16> bf16_tmp[GGML_CUDA_MAX_DEVICES];
-    void * copy_src_ptr[GGML_CUDA_MAX_DEVICES] = {};
-
-    if (use_copy_engine && use_bf16) {
-        to_bf16_cuda_t to_bf16 = ggml_get_to_bf16_cuda(GGML_TYPE_F32);
-        for (int i = 0; i < n; ++i) {
-            auto * cuda_ctx = static_cast<ggml_backend_cuda_context *>(backends[i]->context);
-            GGML_ASSERT(cuda_ctx->device == p->devices[i]);
-            bf16_tmp[i].pool = &cuda_ctx->pool();
-            bf16_tmp[i].alloc(ne);
-            ggml_cuda_set_device(p->devices[i]);
-            if (compute_flag[i]) {
-                to_bf16(tensors[i]->data, bf16_tmp[i].get(), ne, cuda_ctx->stream());
-                CUDA_CHECK(cudaGetLastError());
-            } else {
-                CUDA_CHECK(cudaMemsetAsync(bf16_tmp[i].get(), 0, nbytes, cuda_ctx->stream()));
-            }
-            copy_src_ptr[i] = bf16_tmp[i].get();
-        }
-    }
-
-    bool ok = true;
-    if (use_copy_engine) {
-        // After up-front BF16 conversion, the tmp buffers already hold the
-        // (possibly zeroed-for-inactive) data, so the inner path can treat
-        // every shard as compute.
-        bool inner_compute[GGML_CUDA_MAX_DEVICES];
-        for (int i = 0; i < n; ++i) {
-            inner_compute[i] = use_bf16 ? true : compute_flag[i];
-        }
-
-        // Dispatch into copy_impl with explicit src/dst types.  When use_bf16
-        // is on, the wire type is BF16 (src = bf16_tmp) and the accumulator
-        // is F32 (dst = tensors[i]->data); the combined add kernel rounds dst
-        // through BF16 for bit-equivalence and writes F32 directly, so no
-        // post-conversion is needed.  Otherwise src == dst (same native type).
-        if (use_bf16) {
-            GGML_ASSERT(kernel_type == GGML_TYPE_BF16);
-            nv_bfloat16 * src[GGML_CUDA_MAX_DEVICES] = {};
-            float       * dst[GGML_CUDA_MAX_DEVICES] = {};
-            for (int i = 0; i < n; ++i) {
-                src[i] = static_cast<nv_bfloat16 *>(copy_src_ptr[i]);
-                dst[i] = static_cast<float *>(tensors[i]->data);
-            }
-            ok = ggml_cuda_ar_allreduce_copy_outer<nv_bfloat16, float>(
-                p, backends, src, dst, inner_compute, ne);
-        } else {
-            switch (kernel_type) {
-                case GGML_TYPE_F32: {
-                    float * buf[GGML_CUDA_MAX_DEVICES] = {};
-                    for (int i = 0; i < n; ++i) {
-                        buf[i] = static_cast<float *>(tensors[i]->data);
-                    }
-                    ok = ggml_cuda_ar_allreduce_copy_outer<float, float>(
-                        p, backends, buf, buf, inner_compute, ne);
-                    break;
-                }
-                case GGML_TYPE_BF16: {
-                    nv_bfloat16 * buf[GGML_CUDA_MAX_DEVICES] = {};
-                    for (int i = 0; i < n; ++i) {
-                        buf[i] = static_cast<nv_bfloat16 *>(tensors[i]->data);
-                    }
-                    ok = ggml_cuda_ar_allreduce_copy_outer<nv_bfloat16, nv_bfloat16>(
-                        p, backends, buf, buf, inner_compute, ne);
-                    break;
-                }
-                case GGML_TYPE_F16: {
-                    half * buf[GGML_CUDA_MAX_DEVICES] = {};
-                    for (int i = 0; i < n; ++i) {
-                        buf[i] = static_cast<half *>(tensors[i]->data);
-                    }
-                    ok = ggml_cuda_ar_allreduce_copy_outer<half, half>(
-                        p, backends, buf, buf, inner_compute, ne);
-                    break;
-                }
-                default:
-                    GGML_ASSERT(false);
-            }
-        }
-    } else {
-        // host_buf carries T_wire-typed data; max_chunk_elems is the count that
-        // fits in one host_buf at the wire size.
-        const size_t max_chunk_elems = p->buf_bytes / type_size;
-        const size_t input_type_size = ggml_type_size(input_type);
-
-        // Chunked kernel path runs entirely on the caller's compute stream:
-        // since AR is a barrier here, same-stream ordering subsumes any
-        // cross-stream event handshake that the copy-engine path needs, and
-        // skips the cross-stream scheduling overhead that was hurting the
-        // small-tensor (tg) latency on the AR-stream variant.  Only ev.ker is
-        // still recorded at end-of-AR for acquire_slot's pool-wraparound check.
-        for (int64_t chunk_start = 0; chunk_start < ne; chunk_start += (int64_t) max_chunk_elems) {
-            const size_t remaining_elems = (size_t) (ne - chunk_start);
-            const size_t chunk_elems = remaining_elems < max_chunk_elems ? remaining_elems : max_chunk_elems;
-            const size_t chunk_dst_bytes  = chunk_elems * input_type_size;
-
-            const auto [slot, token] = ggml_cuda_ar_acquire_slot(p);
-            const bool last_chunk = chunk_start + (int64_t) chunk_elems == ne;
-
-            for (int i = 0; i < n; ++i) {
-                const int peer = 1 - i;  // valid for n == 2 only
-                ggml_cuda_set_device(p->devices[i]);
-                auto * cuda_ctx = static_cast<ggml_backend_cuda_context *>(backends[i]->context);
-                GGML_ASSERT(cuda_ctx->device == p->devices[i]);
-                cudaStream_t stream = cuda_ctx->stream();
-
-                char * data = static_cast<char *>(tensors[i]->data) + chunk_start * (int64_t) input_type_size;
-
-                // Match NCCL/meta-backend semantics: inactive shards contribute
-                // zeros.  On the BF16 path the F32 tensor data was already
-                // zeroed up-front (above), so per-chunk zeroing isn't needed.
-                if (!compute_flag[i] && !use_bf16) {
-                    CUDA_CHECK(cudaMemsetAsync(data, 0, chunk_dst_bytes, stream));
-                }
-
-#define LAUNCH_AR_KERNEL(T_dst, T_wire) \
-                ggml_cuda_ar_kernel<T_dst, T_wire><<<dim3(GGML_CUDA_AR_KERNEL_BLOCKS), dim3(256), 0, stream>>>( \
-                    reinterpret_cast<const T_dst *>(data), \
-                    reinterpret_cast<T_dst *>(data), \
-                    reinterpret_cast<T_wire *>(p->host_buf[i].dev + (size_t) slot * p->buf_bytes), \
-                    reinterpret_cast<const T_wire *>(p->host_buf[peer].dev + (size_t) slot * p->buf_bytes), \
-                    static_cast<int>(chunk_elems), \
-                    ggml_cuda_ar_arrival_ptr(p, slot, i), \
-                    ggml_cuda_ar_arrival_ptr(p, slot, peer), \
-                    token)
-
-                if (use_bf16) {
-                    GGML_ASSERT(input_type == GGML_TYPE_F32);
-                    LAUNCH_AR_KERNEL(float, nv_bfloat16);
-                } else {
-                    switch (input_type) {
-                        case GGML_TYPE_F32:  LAUNCH_AR_KERNEL(float,       float);       break;
-                        case GGML_TYPE_F16:  LAUNCH_AR_KERNEL(half,        half);        break;
-                        case GGML_TYPE_BF16: LAUNCH_AR_KERNEL(nv_bfloat16, nv_bfloat16); break;
-                        default: GGML_ASSERT(false);
-                    }
-                }
-
-#undef LAUNCH_AR_KERNEL
-                CUDA_CHECK(cudaGetLastError());
-
-                if (last_chunk) {
-                    CUDA_CHECK(cudaEventRecord(p->ev_pool[i][slot].ker, stream));
-                }
-            }
-        }
-    }
-
-    return ok;
-}
-
-#else // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
-
-// HIP and MUSA lack the host-mapped pinned-memory APIs (cudaHostAllocPortable
-// / cudaHostAllocMapped / cudaHostGetDevicePointer) and __nanosleep that this
-// implementation relies on, so the internal AllReduce is a CUDA-only feature.
-// The dispatcher in ggml-cuda.cu treats a nullptr pipeline as "init failed"
-// and silently falls back to the meta backend's generic AllReduce.
-ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(const int *, size_t) {
-    return nullptr;
-}
-void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline *) {
-}
-bool ggml_cuda_ar_allreduce(ggml_cuda_ar_pipeline *, ggml_backend_t *, ggml_tensor **) {
-    return false;
-}
-
-#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
@@ -1,29 +0,0 @@
-#pragma once
-
-#include "common.cuh"
-#include "ggml-backend-impl.h"
-
-#include <cstddef>
-
-// Opaque pipeline context -- owns all pinned buffers, streams, and events.
-struct ggml_cuda_ar_pipeline;
-
-// Allocate a pipeline for n_devices GPUs.
-// devices[] holds the CUDA device IDs in rank order.
-// Returns nullptr on allocation failure.
-ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(
-    const int * devices, size_t n_devices);
-
-// Release all resources owned by the pipeline.
-void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * pipeline);
-
-// Execute an in-place AllReduce (sum) across tensors[0..n_devices-1].
-// tensors[i] must live on the device managed by backends[i] and be
-// contiguous F32, F16, or BF16.
-// Preconditions are checked by the CUDA comm dispatcher before calling this.
-// Returns true once the reduction work has been enqueued successfully.
-bool ggml_cuda_ar_allreduce(
-    ggml_cuda_ar_pipeline * pipeline,
-    ggml_backend_t        * backends,
-    ggml_tensor           ** tensors);
-
@@ -4,7 +4,6 @@
 #    include <cub/cub.cuh>
 #    if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1)
 #        define STRIDED_ITERATOR_AVAILABLE
-#        include <cuda/iterator>
 #    endif
 using namespace cub;
 #endif  // GGML_CUDA_USE_CUB
@@ -61,11 +61,6 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2,  64,  64,  64,  64, 2, true);
    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 128, 2,  64,  64,  64,  64, 2, true);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128,  8,  64, 4,  64,  96,  64,  64, 2, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16,  64, 4,  32,  96,  64,  64, 2, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 128, 2,  32,  96,  64,  64, 2, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 128, 2,  32,  96,  64,  64, 2, true);
-
    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8,  64, 4,  64, 128, 128, 128, 2, true);
    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16,  64, 4,  32, 128, 128, 128, 2, true);
    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  32, 128, 128, 128, 2, true);
@@ -125,107 +120,61 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
 }

 static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_rdna(const int DKQ, const int DV, const int ncols) {
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64,  8, 128, 2,  64,  32,  32,  32, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 16, 128, 2,  64,  32,  32,  32, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 32, 128, 2,  64,  32,  32,  32, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 64, 128, 2,  64,  32,  32,  32, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 128, 2,  64, 128, 128, 128, 2, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  64, 128, 128,  64, 2, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2,  64, 128, 128,  64, 2, true);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80,  8,  64, 2,  32,  40,  40,  40, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 16,  64, 2,  32,  40,  40,  40, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 32, 128, 2,  64,  40,  40,  40, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 64, 128, 2,  64,  40,  40,  40, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2,  64, 160, 128,  64, 2, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2,  64, 160, 128,  64, 2, false);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96,  8,  64, 2,  32,  48,  48,  48, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 16,  64, 2,  32,  48,  48,  48, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 32, 128, 2,  64,  48,  48,  48, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 64, 128, 2,  64,  48,  48,  48, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16,  64, 4,  32, 128, 128, 128, 1, false);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 128, 2,  32, 128, 128, 128, 1, false);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 64, 256, 1,  32, 128, 128, 128, 1, false);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112,  8,  64, 2,  32,  56,  56,  56, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16,  64, 2,  32,  56,  56,  56, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 128, 2,  64,  56,  56,  56, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 64, 128, 2,  64,  56,  56,  56, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16,  64, 4,  32,  96,  64, 128, 1, false);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2,  32, 160, 128, 128, 1, false);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1,  32, 160, 128, 128, 1, false);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128,  8,  64, 2,  32,  64,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16,  64, 2,  32,  64,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2,  64,  64,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 128, 2,  64,  64,  64,  64, 1, true);
-
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128,  8,  64, 2,  32,  96,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16,  64, 2,  32,  96,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 128, 2,  64,  96,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 128, 2,  64,  96,  64,  64, 1, true);
-
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8,  64, 2,  32, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16,  64, 2,  32, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  64, 128, 128,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2,  64, 128, 128,  64, 1, true);
-
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2,  32, 160, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2,  32, 160, 128, 128, 1, true);
-
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512,  8, 128, 3,  64,  96,  64, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 128, 3,  64,  96,  64, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 128, 2,  32, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 64, 128, 2,  32, 128, 128, 128, 1, true);
-
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512,  8, 128, 3,  64,  96,  64, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 128, 3,  64,  96,  64, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2,  32, 160, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 128, 2,  32, 160, 128, 128, 1, true);
-
-    return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false);
+    // TODO tune specifically for RDNA
+    return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);
 }

 static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_cdna(const int DKQ, const int DV, const int ncols) {
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64,  8, 128, 1,  64,  32,  32,  32, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 16, 256, 2,  64,  32,  32,  32, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 32, 256, 2,  64,  32,  32,  32, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 64, 256, 4,  64,  32,  32,  32, 1, true);
+    // Conservative configs for CDNA (MI100+): 64KB LDS, wavefront64, nstages=1 (no cp.async).
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64,  8, 128, 2, 128,  32,  32,  32, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 16, 128, 2,  64,  32,  32,  32, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 32, 128, 2,  64,  32,  32,  32, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 64, 256, 2,  64,  32,  32,  32, 1, true);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80,  8, 256, 2,  64,  40,  40,  40, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 16, 256, 2,  64,  40,  40,  40, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 32, 256, 2,  64,  40,  40,  40, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80,  8, 128, 2, 128,  40,  40,  40, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 16, 128, 2,  64,  40,  40,  40, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 32, 128, 2,  64,  40,  40,  40, 1, true);
    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 64, 256, 2,  64,  40,  40,  40, 1, true);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96,  8, 256, 2,  64,  48,  48,  48, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 16, 256, 2,  64,  48,  48,  48, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 32, 256, 2,  64,  48,  48,  48, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96,  8, 128, 2, 128,  48,  48,  48, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 16, 128, 2,  64,  48,  48,  48, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 32, 128, 2,  64,  48,  48,  48, 1, true);
    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 64, 256, 2,  64,  48,  48,  48, 1, true);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112,  8, 256, 2,  64,  56,  56,  56, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 256, 2,  64,  56,  56,  56, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 256, 2,  64,  56,  56,  56, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112,  8, 128, 2, 128,  56,  56,  56, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 128, 2,  64,  56,  56,  56, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 128, 2,  64,  56,  56,  56, 1, true);
    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 64, 256, 2,  64,  56,  56,  56, 1, true);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128,  8, 256, 2,  64,  64,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 256, 2,  64,  64,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 256, 2,  64,  64,  64,  64, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128,  8, 128, 2, 128,  64,  64,  64, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 128, 2,  64,  64,  64,  64, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2,  64,  64,  64,  64, 1, true);
    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 256, 2,  64,  64,  64,  64, 1, true);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128,  8, 256, 1,  64,  64,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16, 256, 1,  64,  64,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 256, 1,  64,  64,  64,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 512, 1,  64,  64,  64,  64, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8,  64, 4,  64, 128, 128, 128, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16,  64, 4,  32, 128, 128, 128, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  32, 128, 128, 128, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2,  32, 128, 128, 128, 1, true);

-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8, 256, 1,  64, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 256, 1,  64, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 256, 1,  64, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 512, 1,  64, 128, 128,  64, 1, true);
-
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 256, 1,  64, 160, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 256, 1,  64, 160, 128, 128, 1, true);
-
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512,  8, 256, 1,  64, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 256, 1,  64, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 256, 1,  64, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 64, 256, 1,  64, 128, 128, 128, 1, true);
-
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512,  8, 256, 1,  64, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 256, 1,  64, 128, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 256, 1,  64, 160, 128, 128, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1,  64, 160, 128, 128, 1, true);
-
-    return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false);
+    // Fallback for unsupported DKQ values (e.g. 576). Must return non-zero values to satisfy
+    // compile-time static_asserts even though the kernel guard prevents runtime execution.
+    // nthreads=256 gives nwarps=4 (warp_size=64) or 8 (warp_size=32), nbatch_fa=128 satisfies np*16 divisibility.
+    return fattn_mma_config(256, 1, 128, 4, 4, 4, 1, false);
 }

 static __host__ fattn_mma_config ggml_cuda_fattn_mma_get_config(const int DKQ, const int DV, const int ncols, const int cc) {
@@ -556,7 +505,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
        const int jt,
        const int kb0,
        const int k_VKQ_sup) {
-#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
+#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)
    constexpr int  warp_size       = ggml_cuda_get_physical_warp_size();
    constexpr int  ncols           = ncols1 * ncols2;
    constexpr int  cols_per_warp   = T_B_KQ::I;
@@ -758,18 +707,6 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
 #pragma unroll
            for (int i00 = 0; i00 < nbatch_fa; i00 += np*T_C_KQ::J) {
                const int i0 = i00 + (threadIdx.y % np)*T_C_KQ::J;
-
-                // The mask is stored as 16 bit half values, loading them as 32 bit half2 values is preferred in terms of speed.
-                // However, this is not possible for RDNA3 where 2 consecutive l indices are not consecutive in the mask memory layout.
-#ifdef RDNA3
-#pragma unroll
-                for (int l = 0; l < T_C_KQ::ne; ++l) {
-                    const int i = i0 + T_C_KQ::get_j(l);
-                    const int j = ((threadIdx.y / np)*cols_per_warp + T_C_KQ::get_i(l)) / ncols2;
-
-                    KQ_C[i00/(np*T_C_KQ::J)].x[l] += __half2float(tile_mask[j*(nbatch_fa + 8) + i]);
-                }
-#else
 #pragma unroll
                for (int l0 = 0; l0 < T_C_KQ::ne; l0 += 2) {
                    const int i = (i0 + T_C_KQ::get_j(l0)) / 2;
@@ -779,7 +716,6 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
                    KQ_C[i00/(np*T_C_KQ::J)].x[l0 + 0] += slope*tmp.x;
                    KQ_C[i00/(np*T_C_KQ::J)].x[l0 + 1] += slope*tmp.y;
                }
-#endif // RDNA3
            }
        }

@@ -886,23 +822,13 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
            }
        }
 #elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
-        if constexpr (std::is_same_v<decltype(T_C_VKQ::x), half2[T_C_VKQ::ne]>) {
-            const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]);
+        const half2 KQ_max_scale_h2 = make_half2(
+            KQ_max_scale[0], KQ_max_scale[0]);
 #pragma unroll
-            for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {
+        for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {
 #pragma unroll
-                for (int l = 0; l < T_C_VKQ::ne; ++l) {
-                    VKQ_C[i].x[l] *= KQ_max_scale_h2;
-                }
-            }
-        } else {
-            static_assert(std::is_same_v<decltype(T_C_VKQ::x), float[T_C_VKQ::ne]>, "bad VKQ type");
-#pragma unroll
-            for (int i = 0; i < DV/T_C_VKQ::J; ++i) {
-#pragma unroll
-                for (int l = 0; l < T_C_VKQ::ne; ++l) {
-                    VKQ_C[i].x[l] *= KQ_max_scale[0];
-                }
+            for (int l = 0; l < T_C_VKQ::ne; ++l) {
+                VKQ_C[i].x[l] *= KQ_max_scale_h2;
            }
        }
 #else // Volta
@@ -970,8 +896,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
        const half2 * tile_V_i = !V_is_K_view || i0_stop > 2*nbatch_K2 ? tile_V : tile_V + i0_start/2;

 #if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
+        constexpr int i0_stride = cols_per_warp == 8 ? T_C_VKQ::I : 2*T_C_VKQ::J;
 #pragma unroll
-        for (int i_VKQ_0 = i0_start; i_VKQ_0 < i0_stop; i_VKQ_0 += T_A_VKQ::I) {
+        for (int i_VKQ_0 = i0_start; i_VKQ_0 < i0_stop; i_VKQ_0 += i0_stride) {
            static_assert((nbatch_fa/2) % (np*T_A_VKQ::J) == 0, "bad loop size");
 #pragma unroll
            for (int k00 = 0; k00 < nbatch_fa/2; k00 += np*T_A_VKQ::J) {
@@ -980,15 +907,15 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
                T_A_VKQ A; // Transposed in SRAM but not in registers, gets transposed on load.
                load_ldmatrix_trans(A, tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);
                if constexpr (T_B_KQ::I == 8) {
-                    mma(VKQ_C[i_VKQ_0/T_A_VKQ::I], A, B[k00/(np*T_A_VKQ::J)]);
+                    mma(VKQ_C[i_VKQ_0/i0_stride], A, B[k00/(np*T_A_VKQ::J)]);
                } else {
                    // Wide version of VKQ_C is column-major.
 #if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
                    // AMD matrix C is column-major.
-                    mma(VKQ_C[i_VKQ_0/T_A_VKQ::I], A, B[k00/(np*T_A_VKQ::J)]);
+                    mma(VKQ_C[i_VKQ_0/i0_stride], A, B[k00/(np*T_A_VKQ::J)]);
 #else
                    // swap A and B for CUDA.
-                    mma(VKQ_C[i_VKQ_0/T_A_VKQ::I], B[k00/(np*T_A_VKQ::J)], A);
+                    mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::J)], A);
 #endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
                }
            }
@@ -1021,11 +948,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
        tile_Q, tile_K, tile_V, tile_mask,
        Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0);
    NO_DEVICE_CODE;
-#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
+#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)
 }

 #if defined(TURING_MMA_AVAILABLE)
-template<int DV, int ncols> struct mma_tile_sizes {
+template<int ncols> struct mma_tile_sizes {
    using T_A_KQ  = tile<16,  8, half2>; // row-major
    using T_B_KQ  = tile<16,  8, half2>; // column-major
    using T_C_KQ  = tile<16, 16, float>; // column-major
@@ -1033,7 +960,7 @@ template<int DV, int ncols> struct mma_tile_sizes {
    using T_B_VKQ = tile<16,  8, half2>; // column-major
    using T_C_VKQ = tile<16,  8, half2>; // column-major
 };
-template<int DV> struct mma_tile_sizes<DV, 8> {
+template<> struct mma_tile_sizes<8> {
    using T_A_KQ  = tile<16,  8, half2>; // row-major
    using T_B_KQ  = tile< 8,  8, half2>; // column-major
    using T_C_KQ  = tile<16,  8, float>; // row-major
@@ -1041,60 +968,8 @@ template<int DV> struct mma_tile_sizes<DV, 8> {
    using T_B_VKQ = tile< 8,  8, half2>; // column-major
    using T_C_VKQ = tile<16,  4, half2>; // row-major
 };
-#elif defined(AMD_WMMA_AVAILABLE)
-#ifdef RDNA3
-template<int DV, int ncols> struct mma_tile_sizes {
-    using T_A_KQ  = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major
-    using T_B_KQ  = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major
-    using T_C_KQ  = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>;          // column-major
-    using T_A_VKQ = tile<32,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major
-    using T_B_VKQ = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major
-    using T_C_VKQ = tile<16, 16, half2, DATA_LAYOUT_I_MAJOR>;          // column-major
-};
-template<int ncols> struct mma_tile_sizes<80, ncols> {
-    using T_A_KQ  = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major
-    using T_B_KQ  = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major
-    using T_C_KQ  = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>;          // column-major
-    using T_A_VKQ = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major
-    using T_B_VKQ = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major
-    using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>;          // column-major
-};
-template<int ncols> struct mma_tile_sizes<112, ncols> {
-    using T_A_KQ  = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major
-    using T_B_KQ  = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major
-    using T_C_KQ  = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>;          // column-major
-    using T_A_VKQ = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major
-    using T_B_VKQ = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major
-    using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>;          // column-major
-};
-#else
-template<int DV, int ncols> struct mma_tile_sizes {
-    using T_A_KQ  = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR>;           // row-major
-    using T_B_KQ  = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR>;           // column-major
-    using T_C_KQ  = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>;           // column-major
-    using T_A_VKQ = tile<32,  8, half2, DATA_LAYOUT_I_MAJOR>;           // row-major
-    using T_B_VKQ = tile<16,  8, half2, DATA_LAYOUT_I_MAJOR>;           // column-major
-    using T_C_VKQ = tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED>; // column-major
-};
-template<int ncols> struct mma_tile_sizes<80, ncols> {
-    using T_A_KQ  = tile<16,  8, half2>; // row-major
-    using T_B_KQ  = tile<16,  8, half2>; // column-major
-    using T_C_KQ  = tile<16, 16, float>; // column-major
-    using T_A_VKQ = tile<16,  8, half2>; // row-major
-    using T_B_VKQ = tile<16,  8, half2>; // column-major
-    using T_C_VKQ = tile<16,  8, half2>; // column-major
-};
-template<int ncols> struct mma_tile_sizes<112, ncols> {
-    using T_A_KQ  = tile<16,  8, half2>; // row-major
-    using T_B_KQ  = tile<16,  8, half2>; // column-major
-    using T_C_KQ  = tile<16, 16, float>; // column-major
-    using T_A_VKQ = tile<16,  8, half2>; // row-major
-    using T_B_VKQ = tile<16,  8, half2>; // column-major
-    using T_C_VKQ = tile<16,  8, half2>; // column-major
-};
-#endif // RDNA3
-#elif defined(AMD_MFMA_AVAILABLE)
-template<int DV, int ncols> struct mma_tile_sizes {
+#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
+template<int ncols> struct mma_tile_sizes {
    using T_A_KQ  = tile<16,  8, half2>; // row-major
    using T_B_KQ  = tile<16,  8, half2>; // column-major
    using T_C_KQ  = tile<16, 16, float>; // column-major
@@ -1103,7 +978,7 @@ template<int DV, int ncols> struct mma_tile_sizes {
    using T_C_VKQ = tile<16,  8, half2>; // column-major
 };
 #else // Volta
-template<int DV, int ncols> struct mma_tile_sizes {
+template<int ncols> struct mma_tile_sizes {
    using T_A_KQ  = tile< 8,  4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major
    using T_B_KQ  = tile<32,  4, half2, DATA_LAYOUT_I_MAJOR>;          // column-major
    using T_C_KQ  = tile<32,  8, float, DATA_LAYOUT_I_MAJOR>;          // column-major
@@ -1138,17 +1013,17 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
        const int zt_gqa,
        const int kb0_start,
        const int kb0_stop) {
-#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
+#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)
    //In this kernel Q, K, V are matrices while i, j, k are matrix indices.

    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
    constexpr int ncols = ncols1 * ncols2;
-    using     T_A_KQ    = typename mma_tile_sizes<DV, ncols>::T_A_KQ;
-    using     T_B_KQ    = typename mma_tile_sizes<DV, ncols>::T_B_KQ;
-    using     T_C_KQ    = typename mma_tile_sizes<DV, ncols>::T_C_KQ;
-    using     T_A_VKQ   = typename mma_tile_sizes<DV, ncols>::T_A_VKQ;
-    using     T_B_VKQ   = typename mma_tile_sizes<DV, ncols>::T_B_VKQ;
-    using     T_C_VKQ   = typename mma_tile_sizes<DV, ncols>::T_C_VKQ;
+    using     T_A_KQ    = typename mma_tile_sizes<ncols>::T_A_KQ;
+    using     T_B_KQ    = typename mma_tile_sizes<ncols>::T_B_KQ;
+    using     T_C_KQ    = typename mma_tile_sizes<ncols>::T_C_KQ;
+    using     T_A_VKQ   = typename mma_tile_sizes<ncols>::T_A_VKQ;
+    using     T_B_VKQ   = typename mma_tile_sizes<ncols>::T_B_VKQ;
+    using     T_C_VKQ   = typename mma_tile_sizes<ncols>::T_C_VKQ;

    constexpr int  cols_per_warp   = T_B_KQ::I;
    constexpr int  cols_per_thread = get_cols_per_thread();
@@ -1181,8 +1056,6 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
    T_B_KQ    Q_B[(Q_in_reg ? DKQ/(2*T_B_KQ::J) : 1)];
 #if defined(TURING_MMA_AVAILABLE)
    T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)];
-#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
-    T_C_VKQ VKQ_C[DV % 32 != 0       ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)];
 #elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
    T_C_VKQ VKQ_C[                                     DV/(2*T_C_VKQ::J)];
 #else // Volta
@@ -1391,23 +1264,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
            }
        }
 #elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
-        if constexpr (std::is_same_v<decltype(T_C_VKQ::x), half2[T_C_VKQ::ne]>) {
-            const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]);
+        const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]);
 #pragma unroll
-            for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {
+        for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {
 #pragma unroll
-                for (int l = 0; l < T_C_VKQ::ne; ++l) {
-                    VKQ_C[i].x[l] *= KQ_max_scale_h2;
-                }
-            }
-        } else {
-            static_assert(std::is_same_v<decltype(T_C_VKQ::x), float[T_C_VKQ::ne]>, "bad VKQ type");
-#pragma unroll
-            for (int i = 0; i < DV/T_C_VKQ::J; ++i) {
-#pragma unroll
-                for (int l = 0; l < T_C_VKQ::ne; ++l) {
-                    VKQ_C[i].x[l] *= KQ_max_scale[0];
-                }
+            for (int l = 0; l < T_C_VKQ::ne; ++l) {
+                VKQ_C[i].x[l] *= KQ_max_scale_h2;
            }
        }
 #else // Volta
@@ -1566,7 +1428,6 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
 #pragma unroll
    for (int k00 = 0; k00 < DV/2; k00 += nbatch_combine) {
        if constexpr (cols_per_warp == 8) {
-            static_assert(std::is_same_v<decltype(T_C_VKQ::x), half2[T_C_VKQ::ne]>, "bad VKQ type");
            const int jc_cwd = threadIdx.y*T_B_KQ::I + T_B_KQ::get_i(-1); // jc combine write data
 #pragma unroll
            for (int k1 = 0; k1 < nbatch_combine; k1 += T_B_KQ::J) {
@@ -1581,45 +1442,14 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
            }
        } else {
            const int j0 = threadIdx.y*cols_per_warp;
-            if constexpr (std::is_same_v<decltype(T_C_VKQ::x), half2[T_C_VKQ::ne]>) {
-                if constexpr (T_C_VKQ::dl == DATA_LAYOUT_I_MAJOR) {
 #pragma unroll
-                    for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J) {
+            for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J) {
 #pragma unroll
-                        for (int l = 0; l < T_C_VKQ::ne; ++l) {
-                            const int j = j0 + T_C_VKQ::get_i(l);
-                            const int k = k1 + T_C_VKQ::get_j(l);
+                for (int l = 0; l < T_C_VKQ::ne; ++l) {
+                    const int j = j0 + T_C_VKQ::get_i(l);
+                    const int k = k1 + T_C_VKQ::get_j(l);

-                            tile_Q[j*tile_stride + k] = VKQ_C[(k00 + k1)/T_C_VKQ::J].x[l];
-                        }
-                    }
-                } else {
-                    static_assert(T_C_VKQ::dl == DATA_LAYOUT_I_MAJOR_SCRAMBLED, "bad T_C_VKQ data layout");
-                    using T_C_VKQ_us = tile<T_C_VKQ::I, T_C_VKQ::J, half2, DATA_LAYOUT_I_MAJOR>; // us == unscrambled
-#pragma unroll
-                    for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J) {
-                        const T_C_VKQ_us VKQ_C_us = unscramble(VKQ_C[(k00 + k1)/T_C_VKQ::J]);
-#pragma unroll
-                        for (int l = 0; l < T_C_VKQ_us::ne; ++l) {
-                            const int j = j0 + T_C_VKQ_us::get_i(l);
-                            const int k = k1 + T_C_VKQ_us::get_j(l);
-
-                            tile_Q[j*tile_stride + k] = VKQ_C_us.x[l];
-                        }
-                    }
-                }
-            } else {
-                static_assert(std::is_same_v<decltype(T_C_VKQ::x), float[T_C_VKQ::ne]>, "bad VKQ type");
-                half * tile_Q_h = (half *) tile_Q;
-#pragma unroll
-                for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J/2) {
-#pragma unroll
-                    for (int l = 0; l < T_C_VKQ::ne; ++l) {
-                        const int j = j0 + T_C_VKQ::get_i(l);
-                        const int k = 2*k1 + T_C_VKQ::get_j(l);
-
-                        tile_Q_h[j*(2*tile_stride) + k] = VKQ_C[(k00 + k1)/(T_C_VKQ::J/2)].x[l];
-                    }
+                    tile_Q[j*tile_stride + k] = VKQ_C[(k00 + k1)/T_C_VKQ::J].x[l];
                }
            }
        }
@@ -1697,7 +1527,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
        stride_Q1, stride_Q2, stride_K, stride_V, stride_mask,
        jt, kb0_start, kb0_stop);
    NO_DEVICE_CODE;
-#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
+#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)
 }

 template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view>
@@ -1724,17 +1554,13 @@ static __global__ void flash_attn_ext_f16(
                            const int32_t nb21, const int32_t nb22, const int64_t nb23,
                            const int32_t ne31, const int32_t ne32, const int32_t ne33,
                            const int32_t nb31, const int32_t nb32, const int64_t nb33) {
-#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))
+#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE))

    // Skip unused kernel variants for faster compilation:
    if (use_logit_softcap && !(DKQ == 128 || DKQ == 256 || DKQ == 512)) {
        NO_DEVICE_CODE;
        return;
    }
-    if (DKQ == 192 && ncols2 != 8 && ncols2 != 16) {
-        NO_DEVICE_CODE;
-        return;
-    }
 #ifdef VOLTA_MMA_AVAILABLE
    if (ncols1*ncols2 < 32) {
        NO_DEVICE_CODE;
@@ -1750,14 +1576,14 @@ static __global__ void flash_attn_ext_f16(
 #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING

 #if defined(AMD_WMMA_AVAILABLE)
-    if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 128) {
+    if (ncols1*ncols2 > 32 || ncols1*ncols2 < 16 || DKQ > 128 || ncols2 == 1) {
        NO_DEVICE_CODE;
        return;
    }
 #endif // defined(AMD_WMMA_AVAILABLE)

 #if defined(AMD_MFMA_AVAILABLE)
-    if (ncols1*ncols2 < 16 || DKQ > 256) {
+    if (DKQ != 64 && DKQ != 80 && DKQ != 96 && DKQ != 112 && DKQ != 128) {
        NO_DEVICE_CODE;
        return;
    }
@@ -1880,7 +1706,7 @@ static __global__ void flash_attn_ext_f16(
              ne31, ne32, ne33,
              nb31, nb32, nb33);
    NO_DEVICE_CODE;
-#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))
+#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE))
 }

 template <int DKQ, int DV, int ncols1, int ncols2>
@@ -34,10 +34,6 @@ void ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor
            GGML_ASSERT(V->ne[0] == K->ne[0]);
            ggml_cuda_flash_attn_ext_tile_case<128, 128>(ctx, dst);
        } break;
-        case 192: {
-            GGML_ASSERT(V->ne[0] == 128);
-            ggml_cuda_flash_attn_ext_tile_case<192, 128>(ctx, dst);
-        } break;
        case 256: {
            GGML_ASSERT(V->ne[0] == K->ne[0]);
            ggml_cuda_flash_attn_ext_tile_case<256, 256>(ctx, dst);
@@ -62,12 +62,6 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 2,  64,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2,  64,  64)

-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  2,  64, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  4, 128, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  8, 256, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2,  64,  64)
-
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2,  64, 2,  64,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 128, 2,  64,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 256, 2,  64,  64)
@@ -130,12 +124,6 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 128, 3,  32, 128)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2,  64,  64)

-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  2, 128, 3,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  4, 128, 3,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  8, 256, 2,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2,  32,  64)
-
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2, 128, 3,  64,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 128, 3,  32,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 256, 2,  32, 256)
@@ -205,12 +193,6 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2,  64,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 2,  64,  32)

-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  2, 256, 2, 128,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  4, 256, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  8, 256, 2,  64,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2,  32,  64)
-
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2, 256, 2, 128,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 256, 2,  64, 128)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 256, 2,  64, 128)
@@ -282,12 +264,6 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 3, 128,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 3,  64,  64)

-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  2,  64, 8,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  4, 128, 6,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128,  8, 128, 6,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 5,  32,  64)
-    GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 3,  64,  64)
-
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2,  64, 8,  32,  64)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 128, 6,  32, 256)
    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 128, 6,  32, 256)
@@ -1274,20 +1250,7 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm
        }
    }

-    if constexpr (DKQ == 192) {
-        // MiMo-V2.5 / V2.5-Pro / V2-Flash: gqa_ratio is 8 (SWA) or 16 (full attn)
-        if (use_gqa_opt && gqa_ratio % 16 == 0) {
-            launch_fattn_tile_switch_ncols1<DKQ, DV, 16, use_logit_softcap>(ctx, dst);
-            return;
-        }
-        if (use_gqa_opt && gqa_ratio % 8 == 0) {
-            launch_fattn_tile_switch_ncols1<DKQ, DV,  8, use_logit_softcap>(ctx, dst);
-            return;
-        }
-        GGML_ABORT("flash-attn tile (192/128): expected GQA ratio multiple of 8");
-    }
-
-    if constexpr (DKQ <= 512 && DKQ != 320 && DKQ != 192) {
+    if constexpr (DKQ <= 512 && DKQ != 320) {
        if (use_gqa_opt && gqa_ratio % 8 == 0) {
            launch_fattn_tile_switch_ncols1<DKQ, DV, 8, use_logit_softcap>(ctx, dst);
            return;
@@ -1340,7 +1303,6 @@ extern DECL_FATTN_TILE_CASE( 80,  80);
 extern DECL_FATTN_TILE_CASE( 96,  96);
 extern DECL_FATTN_TILE_CASE(112, 112);
 extern DECL_FATTN_TILE_CASE(128, 128);
-extern DECL_FATTN_TILE_CASE(192, 128);
 extern DECL_FATTN_TILE_CASE(256, 256);
 extern DECL_FATTN_TILE_CASE(320, 256);
 extern DECL_FATTN_TILE_CASE(512, 512);
@@ -19,14 +19,13 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con
    }

    if constexpr (ncols2 <= 16) {
-        if (Q->ne[1] <= 16/ncols2) {
+        if ((turing_mma_available(cc) || amd_wmma_available(cc)) && Q->ne[1] <= 16/ncols2) {
            ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 16/ncols2, ncols2>(ctx, dst);
            return;
        }
    }

-    if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING) ||
-            (GGML_CUDA_CC_IS_AMD(cc) && DKQ > 256)) {
+    if (ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING || amd_wmma_available(cc) || Q->ne[1] <= 32/ncols2) {
        ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 32/ncols2, ncols2>(ctx, dst);
        return;
    }
@@ -140,22 +139,6 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
            GGML_ASSERT(V->ne[0] == 128);
            ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<128, 128>(ctx, dst);
            break;
-        case 192: {
-            // MiMo-V2.5 / V2.5-Pro / V2-Flash: gqa_ratio is 8 (SWA) or 16 (full attn)
-            GGML_ASSERT(V->ne[0] == 128);
-            float max_bias = 0.0f;
-            memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));
-            const bool use_gqa_opt = mask && max_bias == 0.0f;
-            GGML_ASSERT(use_gqa_opt);
-            GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);
-            const int gqa_ratio = Q->ne[2] / K->ne[2];
-            if (gqa_ratio % 16 == 0) {
-                ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<192, 128, 16>(ctx, dst);
-            } else {
-                GGML_ASSERT(gqa_ratio % 8 == 0);
-                ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<192, 128,  8>(ctx, dst);
-            }
-        } break;
        case 256:
            GGML_ASSERT(V->ne[0] == 256);
            ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<256, 256>(ctx, dst);
@@ -385,14 +368,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
                return BEST_FATTN_KERNEL_NONE;
            }
            break;
-        case 192:
-            if (V->ne[0] != 128 || !gqa_opt_applies) {
-                return BEST_FATTN_KERNEL_NONE;
-            }
-            if (gqa_ratio % 8 != 0) {
-                return BEST_FATTN_KERNEL_NONE;
-            }
-            break;
        case 320:
            if (V->ne[0] != 256 || !gqa_opt_applies) {
                return BEST_FATTN_KERNEL_NONE;
@@ -450,8 +425,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
    }

    // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes:
-    // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path.
-    const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0;
+    const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0;

    // If Turing tensor cores are available, use them:
    if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
@@ -478,13 +452,12 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
        return BEST_FATTN_KERNEL_MMA_F16;
    }

-    const int ncols2_max = Q->ne[0] == 320 ? 32 : ((Q->ne[0] == 576 || Q->ne[0] == 192) ? 16 : 8);
-    int gqa_ratio_eff = 1;
-    while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
-        gqa_ratio_eff *= 2;
-    }
-
    if (volta_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
+        int gqa_ratio_eff = 1;
+        const int ncols2_max = Q->ne[0] == 576 ? 16 : 8;
+        while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
+            gqa_ratio_eff *= 2;
+        }
        if (can_use_vector_kernel && Q->ne[1] * gqa_ratio_eff <= 2) {
            return BEST_FATTN_KERNEL_VEC;
        }
@@ -495,29 +468,48 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
    }

    // Use the WMMA kernel if possible:
-    if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 192 && Q->ne[0] != 512 && Q->ne[0] != 576) {
+    if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 512 && Q->ne[0] != 576) {
        if (can_use_vector_kernel && Q->ne[1] <= 2) {
            return BEST_FATTN_KERNEL_VEC;
        }
        return BEST_FATTN_KERNEL_WMMA_F16;
    }

-    // AMD MFMA needs a certain minimum batch size to outscale the tile kernel for large head sizes.
-    if ((amd_mfma_available(cc) && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72) {
-        if ((Q->ne[0] <= 64 && Q->ne[1] * gqa_ratio_eff > 8)) {
-            return BEST_FATTN_KERNEL_MMA_F16;
+    if (amd_wmma_available(cc) && GGML_CUDA_CC_IS_RDNA4(cc) && gqa_opt_applies && Q->ne[0] <= 128 && Q->ne[0] != 40 && Q->ne[0] != 72) {
+        if (can_use_vector_kernel) {
+            if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) {
+                if (Q->ne[1] == 1) {
+                    if (!gqa_opt_applies) {
+                        return BEST_FATTN_KERNEL_VEC;
+                    }
+                }
+            } else {
+                if (Q->ne[1] <= 2) {
+                    return BEST_FATTN_KERNEL_VEC;
+                }
+            }
        }
-        if ((Q->ne[0] <= 128 && Q->ne[1] * gqa_ratio_eff > 16)) {
-            return BEST_FATTN_KERNEL_MMA_F16;
+        int gqa_ratio_eff = 1;
+        const int ncols2_max = Q->ne[0] == 576 ? 16 : 8;
+        while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
+            gqa_ratio_eff *= 2;
        }
-        if ((Q->ne[0] <= 256 && Q->ne[1] * gqa_ratio_eff > 64)) {
-            return BEST_FATTN_KERNEL_MMA_F16;
+        if (Q->ne[1] * gqa_ratio_eff <= 8) {
+            return BEST_FATTN_KERNEL_TILE; // AMD WMMA is only faster if the full tile width of 16 can be utilized.
        }
+        return BEST_FATTN_KERNEL_MMA_F16;
    }

-    // AMD WMMA is always faster than the tile kernel if the full tile width of 16 can be utilized.
-    if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 128) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > 8) {
-        return BEST_FATTN_KERNEL_MMA_F16;
+    // Use MFMA flash attention for CDNA (MI100+):
+    if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 256 && Q->ne[0] != 512 && Q->ne[0] != 576) {
+        const int64_t eff_nq = Q->ne[1] * (gqa_opt_applies ? gqa_ratio : 1);
+        // MMA vs tile crossover benchmarked on MI300X @ d32768:
+        //   hsk=64  (gqa=4): MMA wins at eff >= 128 (+11%)
+        //   hsk=128 (gqa=4): MMA wins at eff >= 128 (+4%)
+        if (eff_nq >= (GGML_CUDA_CC_IS_CDNA1(cc) && Q->ne[0] == 64 ? 64 : 128)) {
+            return BEST_FATTN_KERNEL_MMA_F16;
+        }
+        // Fall through to tile kernel for small effective batch sizes.
    }

    // If there are no tensor cores available, use the generic tile kernel:
@@ -2,7 +2,6 @@
 #include "ggml-impl.h"
 #include "ggml-backend-impl.h"

-#include "ggml-cuda/allreduce.cuh"
 #include "ggml-cuda/common.cuh"
 #include "ggml-cuda/acc.cuh"
 #include "ggml-cuda/add-id.cuh"
@@ -40,7 +39,6 @@
 #include "ggml-cuda/rope.cuh"
 #include "ggml-cuda/roll.cuh"
 #include "ggml-cuda/scale.cuh"
-#include "ggml-cuda/snake.cuh"
 #include "ggml-cuda/softcap.cuh"
 #include "ggml-cuda/softmax.cuh"
 #include "ggml-cuda/ssm-conv.cuh"
@@ -87,9 +85,6 @@

 static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");

-#define GGML_LOG_WARN_ONCE(str) \
-    { static std::once_flag warn_flag; std::call_once(warn_flag, []() { GGML_LOG_WARN(str); }); }
-
 [[noreturn]]
 void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {
    int id = -1; // in case cudaGetDevice fails
@@ -1143,46 +1138,70 @@ static const ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_inte
    /* .is_host          = */ ggml_backend_cuda_split_buffer_type_is_host,
 };

-// Communication context for multi-GPU AllReduce during tensor parallelism.
-//
-// Created once per meta backend instance.  Resources for the selected mode
-// (NCCL communicators or the internal AllReduce pipeline) are initialised
-// eagerly during comm_init so any init failure surfaces at startup rather
-// than mid-run.
-struct ggml_backend_cuda_comm_context {
-    using try_allreduce_fn = bool(*)(ggml_backend_cuda_comm_context *, struct ggml_tensor **);
-
-    std::vector<ggml_backend_t> backends;
-    std::vector<int>            dev_ids;
-
-    // Set by the init chain (comm_init_{nccl, internal, none}) to one of
-    // try_allreduce_{nccl, internal, butterfly}.  nccl needs `comms`,
-    // internal needs `ar_pipeline`, butterfly needs nothing.  Per-call
-    // failures return false; the meta backend's generic implementation then
-    // handles that call.
-    try_allreduce_fn            try_allreduce = nullptr;
-
-    ggml_cuda_ar_pipeline *     ar_pipeline = nullptr;
-
 #ifdef GGML_USE_NCCL
-    std::vector<ncclComm_t>     comms;
-#endif // GGML_USE_NCCL
+struct ggml_backend_cuda_comm_context {
+    std::vector<ggml_backend_t> backends;
+    std::vector<ncclComm_t> comms;

    ~ggml_backend_cuda_comm_context() {
-#ifdef GGML_USE_NCCL
        for (ncclComm_t comm : comms) {
            NCCL_CHECK(ncclCommDestroy(comm));
        }
-#endif // GGML_USE_NCCL
-        ggml_cuda_ar_pipeline_free(ar_pipeline);
    }
 };
+#endif // GGML_USE_NCCL

+static void ggml_backend_cuda_comm_free(void * comm_ctx_v) {
+#ifdef GGML_USE_NCCL
+    if (comm_ctx_v == nullptr) {
+        return;
+    }
+    ggml_backend_cuda_comm_context * comm_ctx = (ggml_backend_cuda_comm_context *) comm_ctx_v;
+    delete comm_ctx;
+#else
+    GGML_UNUSED(comm_ctx_v);
+#endif // GGML_USE_NCCL
+}
+
+static void * ggml_backend_cuda_comm_init(ggml_backend_t * backends, size_t n_backends) {
+#ifdef GGML_USE_NCCL
+    for (size_t i = 0; i < n_backends; i++) {
+        if (!ggml_backend_is_cuda(backends[i])) {
+            return nullptr;
+        }
+    }
+    ggml_backend_cuda_comm_context * ret = new ggml_backend_cuda_comm_context;
+    std::vector<int> dev_ids;
+    ret->backends.reserve(n_backends);
+    dev_ids.reserve(n_backends);
+    for (size_t i = 0; i < n_backends; i++) {
+        ret->backends.push_back(backends[i]);
+        ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backends[i]->context;
+        dev_ids.push_back(cuda_ctx->device);
+    }
+
+    ret->comms.resize(n_backends);
+    NCCL_CHECK(ncclCommInitAll(ret->comms.data(), n_backends, dev_ids.data()));
+    return ret;
+#else
+    // If NCCL is installed it is used by default for optimal performance.
+    // However, NVIDIA does not distribute NCCL with CUDA so users may be unwittingly missing this package.
+    // RCCL is disabled by default, users are explicitly opting in.
+    // Therefore print no warning for RCCL.
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+    static bool warning_printed = false;
+    if (!warning_printed) {
+        GGML_LOG_WARN("%s: NVIDIA Collective Communications Library (NCCL) is unavailable, multi GPU performance will be suboptimal\n", __func__);
+        warning_printed = true;
+    }
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+    GGML_UNUSED_VARS(backends, n_backends);
+    return nullptr;
+#endif // GGML_USE_NCCL
+}
+
+static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) {
 #ifdef GGML_USE_NCCL
-// AllReduce via NCCL. Reduces as FP32 for small tensors and BF16 for large
-// tensors (bandwidth-bound), then converts back to FP32.
-static bool ggml_backend_cuda_comm_allreduce_nccl(
-        ggml_backend_cuda_comm_context * comm_ctx, struct ggml_tensor ** tensors) {
    const int64_t ne = ggml_nelements(tensors[0]);
    // FIXME the input of llm_graph_context::build_in_out_ids can produce a tensor with 0 elements if n_outputs == 0
    // This then causes a crash in this function
@@ -1190,6 +1209,8 @@ static bool ggml_backend_cuda_comm_allreduce_nccl(
        return true;
    }

+    GGML_ASSERT(comm_ctx_v != nullptr);
+    ggml_backend_cuda_comm_context * comm_ctx = (ggml_backend_cuda_comm_context *) comm_ctx_v;
    const size_t n_backends = comm_ctx->backends.size();

    for (size_t i = 0; i < n_backends; ++i) {
@@ -1214,6 +1235,7 @@ static bool ggml_backend_cuda_comm_allreduce_nccl(
            NCCL_CHECK(ncclAllReduce(tensors[i]->data, tensors[i]->data, ne, ncclFloat, ncclSum, comm_ctx->comms[i], cuda_ctx->stream()));
        }
        NCCL_CHECK(ncclGroupEnd());
+
        return true;
    }

@@ -1252,184 +1274,10 @@ static bool ggml_backend_cuda_comm_allreduce_nccl(
    }

    return true;
-}
-#endif // GGML_USE_NCCL
-
-// Run the internal AR pipeline.  Returns false on unsupported / failed input
-// -- the caller decides whether to abort (env-forced) or fall back silently.
-static bool ggml_backend_cuda_comm_allreduce_internal(
-        ggml_backend_cuda_comm_context * comm_ctx, struct ggml_tensor ** tensors) {
-    GGML_ASSERT(comm_ctx->ar_pipeline != nullptr);
-
-    const size_t n_backends = comm_ctx->backends.size();
-    GGML_ASSERT(n_backends == 2);
-    GGML_ASSERT(tensors[0] != nullptr);
-
-    const int64_t   ne   = ggml_nelements(tensors[0]);
-    const ggml_type type = tensors[0]->type;
-
-    if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16 && type != GGML_TYPE_BF16) {
-        GGML_LOG_DEBUG("%s: internal unsupported: type=%d\n", __func__, (int) type);
-        return false;
-    }
-
-    if (ne == 0) {
-        return true;
-    }
-
-    for (size_t i = 0; i < n_backends; ++i) {
-        if (tensors[i] == nullptr) {
-            GGML_LOG_ERROR("%s: internal failed: tensor[%zu] is null\n", __func__, i);
-            return false;
-        }
-        if (ggml_nelements(tensors[i]) != ne || tensors[i]->type != type) {
-            GGML_LOG_ERROR("%s: internal failed: tensor[%zu] ne=%" PRId64 " type=%d expected ne=%" PRId64 " type=%d\n",
-                           __func__, i, ggml_nelements(tensors[i]), (int) tensors[i]->type, ne, (int) type);
-            return false;
-        }
-        if (!ggml_is_contiguously_allocated(tensors[i])) {
-            GGML_LOG_DEBUG("%s: internal unsupported: tensor[%zu] is not contiguously allocated: ne=%" PRId64 " nbytes=%zu packed=%zu type=%d\n",
-                           __func__, i, ne, ggml_nbytes(tensors[i]),
-                           (size_t) ne * ggml_type_size(type) / ggml_blck_size(type), (int) type);
-            return false;
-        }
-        if (((uintptr_t) tensors[i]->data & 0xF) != 0) {
-            GGML_LOG_DEBUG("%s: internal unsupported: tensor[%zu] data pointer is not 16-byte aligned: %p type=%d ne=%" PRId64 "\n",
-                           __func__, i, tensors[i]->data, (int) type, ne);
-            return false;
-        }
-        GGML_ASSERT((ggml_nbytes(tensors[i]) & 0xF) == 0);
-    }
-
-    return ggml_cuda_ar_allreduce(comm_ctx->ar_pipeline, comm_ctx->backends.data(), tensors);
-}
-
-// ---------------------------------------------------------------------------
-// Per-call dispatch -- three variants, one per backend.  Each is set as
-// comm_ctx->try_allreduce by the matching init step.  Per-call failure
-// returns false; the meta backend's generic implementation handles that call.
-// ---------------------------------------------------------------------------
-
-#ifdef GGML_USE_NCCL
-static bool ggml_backend_cuda_comm_try_allreduce_nccl(
-        ggml_backend_cuda_comm_context * comm_ctx, struct ggml_tensor ** tensors) {
-    return ggml_backend_cuda_comm_allreduce_nccl(comm_ctx, tensors);
-}
-#endif // GGML_USE_NCCL
-
-static bool ggml_backend_cuda_comm_try_allreduce_internal(
-        ggml_backend_cuda_comm_context * comm_ctx, struct ggml_tensor ** tensors) {
-    return ggml_backend_cuda_comm_allreduce_internal(comm_ctx, tensors);
-}
-
-static bool ggml_backend_cuda_comm_try_allreduce_butterfly(
-        ggml_backend_cuda_comm_context *, struct ggml_tensor **) {
-    return false;
-}
-
-static void ggml_backend_cuda_comm_free(void * comm_ctx_v) {
-    if (comm_ctx_v == nullptr) {
-        return;
-    }
-    delete static_cast<ggml_backend_cuda_comm_context *>(comm_ctx_v);
-}
-
-// ---------------------------------------------------------------------------
-// Init -- chained nccl -> internal -> none.  Each step tries to bring up its
-// resource; on failure it warns and recurses into the next step.
-// ---------------------------------------------------------------------------
-static void ggml_backend_cuda_comm_init_none(ggml_backend_cuda_comm_context * ret) {
-    ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_butterfly;
-}
-
-static void ggml_backend_cuda_comm_init_internal(ggml_backend_cuda_comm_context * ret) {
-    ret->ar_pipeline = ggml_cuda_ar_pipeline_init(ret->dev_ids.data(), ret->dev_ids.size());
-    if (ret->ar_pipeline) {
-        ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_internal;
-        return;
-    }
-
-    // Clear sticky CUDA error from the failed init.
-    (void) cudaGetLastError();
-    GGML_LOG_WARN("internal AllReduce init failed (n_devices != 2?); "
-                  "falling back to meta-backend butterfly\n");
-    ggml_backend_cuda_comm_init_none(ret);
-}
-
-static void ggml_backend_cuda_comm_init_nccl(ggml_backend_cuda_comm_context * ret) {
-#ifdef GGML_USE_NCCL
-    const size_t n = ret->dev_ids.size();
-    ret->comms.resize(n);
-    ncclResult_t rc = ncclCommInitAll(ret->comms.data(), (int) n, ret->dev_ids.data());
-    if (rc == ncclSuccess) {
-        ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_nccl;
-        return;
-    }
-
-    ret->comms.clear();
-    GGML_LOG_WARN("NCCL init failed (%s); falling back to internal AllReduce\n",
-                  ncclGetErrorString(rc));
-#else // GGML_USE_NCCL
-#ifndef GGML_USE_HIP
-    GGML_LOG_WARN("NCCL not compiled in; falling back to internal AllReduce.  "
-                  "Recompile with -DGGML_CUDA_NCCL=ON for best multi-GPU performance.\n");
-#endif // !GGML_USE_HIP
-#endif // GGML_USE_NCCL
-
-    ggml_backend_cuda_comm_init_internal(ret);
-}
-
-// Top-level init.  Picks one of the three init paths based on
-// GGML_CUDA_ALLREDUCE (or the platform default) and lets the chain handle
-// any fallback.  Unrecognised env values warn and fall through to the
-// platform default.
-static void * ggml_backend_cuda_comm_init(ggml_backend_t * backends, size_t n_backends) {
-    for (size_t i = 0; i < n_backends; i++) {
-        if (!ggml_backend_is_cuda(backends[i])) {
-            return nullptr;
-        }
-    }
-
-    auto * ret = new ggml_backend_cuda_comm_context;
-    ret->backends.assign(backends, backends + n_backends);
-    ret->dev_ids.reserve(n_backends);
-    for (size_t i = 0; i < n_backends; i++) {
-        ret->dev_ids.push_back(static_cast<ggml_backend_cuda_context *>(backends[i]->context)->device);
-    }
-
-    const char * env = getenv("GGML_CUDA_ALLREDUCE");
-    if (!env) {
-        // Platform default: Linux uses NCCL, otherwise (generally Windows) internal
-#if defined(__linux__)
-        ggml_backend_cuda_comm_init_nccl(ret);
 #else
-        ggml_backend_cuda_comm_init_internal(ret);
-#endif // defined(__linux__)
-    } else {
-        std::string env_str(env);
-        if (env_str == "nccl") {
-            ggml_backend_cuda_comm_init_nccl(ret);
-        } else if (env_str == "internal") {
-            ggml_backend_cuda_comm_init_internal(ret);
-        } else if (env_str == "none") {
-            ggml_backend_cuda_comm_init_none(ret);
-        } else {
-            GGML_LOG_WARN("unknown GGML_CUDA_ALLREDUCE value: %s\n", env);
-            ggml_backend_cuda_comm_init_none(ret);
-        }
-    }
-
-    return ret;
-}
-
-// Top-level dispatch -- calls the function pointer chosen by comm_init.
-// Returns false to let the meta-backend's butterfly run.
-static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) {
-    if (comm_ctx_v == nullptr) {
-        return false;
-    }
-    auto * comm_ctx = static_cast<ggml_backend_cuda_comm_context *>(comm_ctx_v);
-    return comm_ctx->try_allreduce(comm_ctx, tensors);
+    GGML_UNUSED_VARS(comm_ctx_v, tensors);
+    return false;
+#endif // GGML_USE_NCCL
 }

 ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) {
@@ -3909,50 +3757,6 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
        return 2;
    }

-    // Snake activation: y = x + sin(a*x)^2 * inv_b
-    // Naive 5-op decomposition emitted by frontends: mul -> sin -> sqr -> mul -> add
-    if (ggml_can_fuse_subgraph(cgraph, i,
-            { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD },
-            { i + 4 })) {
-        const ggml_tensor * mul0 = cgraph->nodes[i];
-        const ggml_tensor * sqr  = cgraph->nodes[i + 2];
-        const ggml_tensor * mul1 = cgraph->nodes[i + 3];
-        ggml_tensor *       add  = cgraph->nodes[i + 4];
-
-        // x carries the full activation shape, a is the broadcast operand
-        const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1];
-        const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0];
-
-        // mul1 reads sqr and inv_b in either operand order
-        const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0];
-
-        // closure check: the trailing add must read the same x as the leading mul
-        const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0];
-
-        // Kernel iterates over total = T * C, so x and add must be 2D and
-        // a / inv_b must collapse to [1, C, 1, 1]. Higher dims are not handled.
-        const bool dim_ok   = (x->ne[2]   == 1 && x->ne[3]   == 1) &&
-                              (add->ne[2] == 1 && add->ne[3] == 1) &&
-                              (a->ne[2]   == 1 && a->ne[3]   == 1);
-        const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1];
-
-        // x must be in the supported whitelist and every operand / intermediate
-        // result must share x's type, since launch_snake casts a / inv_b as
-        // float and templates the kernel on a single T. Mixed precision chains
-        // fall back to the naive path.
-        const ggml_tensor * sin1 = cgraph->nodes[i + 1];
-        const bool types_ok = (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) &&
-                              (a->type    == x->type) && (inv_b->type == x->type) &&
-                              (mul0->type == x->type) && (sin1->type  == x->type) &&
-                              (sqr->type  == x->type) && (mul1->type  == x->type) &&
-                              (add->type  == x->type);
-
-        if (types_ok && shape_ok && dim_ok && x_in_add == x) {
-            ggml_cuda_op_snake_fused(*cuda_ctx, x, a, inv_b, add);
-            return 4;
-        }
-    }
-
    // multi-(add or mul)
    if (node->op == GGML_OP_ADD || node->op == GGML_OP_MUL) {
        int     n_fuse = 0;
@@ -5306,8 +5110,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
        case GGML_OP_VIEW:
        case GGML_OP_PERMUTE:
        case GGML_OP_TRANSPOSE:
+        case GGML_OP_ADD:
        case GGML_OP_ADD_ID:
        case GGML_OP_ADD1:
+        case GGML_OP_SUB:
+        case GGML_OP_MUL:
+        case GGML_OP_DIV:
        case GGML_OP_SCALE:
        case GGML_OP_SQR:
        case GGML_OP_SQRT:
@@ -5316,13 +5124,6 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
        case GGML_OP_CLAMP:
        case GGML_OP_LOG:
            return true;
-        case GGML_OP_ADD:
-        case GGML_OP_SUB:
-        case GGML_OP_MUL:
-        case GGML_OP_DIV:
-            return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
-                   (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) &&
-                   (op->type         == GGML_TYPE_F32 || op->type         == GGML_TYPE_F16);
        case GGML_OP_SSM_SCAN: {
            if (op->src[3]->ne[0] == 1) {
                // Mamba2
@@ -5633,9 +5434,6 @@ ggml_backend_reg_t ggml_backend_cuda_reg() {
                char pci_bus_id[32] = {};
                CUDA_CHECK(cudaDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), i));
                dev_ctx->pci_bus_id = pci_bus_id;
-                for (char & c : dev_ctx->pci_bus_id) {
-                    c = std::tolower(c);
-                }
                dev_ctx->op_offload_min_batch_size = min_batch_size;

                ggml_backend_dev_t dev = new ggml_backend_device {
@@ -1,6 +1,5 @@
 #include "im2col.cuh"

-#define MAX_GRIDDIM_Y 65535
 #define MAX_GRIDDIM_Z 65535

 template <typename T>
@@ -19,23 +18,22 @@ static  __global__ void im2col_kernel(
    const int64_t ikh = rem / KW;
    const int64_t ikw = rem - ikh * KW;

-    for (int64_t iow = blockIdx.y; iow < OW; iow += MAX_GRIDDIM_Y) {
-        for (int64_t iz = blockIdx.z; iz < N_OH; iz += MAX_GRIDDIM_Z) {
-            const int64_t  in = iz / OH;
-            const int64_t  ioh = iz - in * OH;
+    const int64_t  iow = blockIdx.y;
+    for (int64_t iz = blockIdx.z; iz < N_OH; iz+=MAX_GRIDDIM_Z) {
+        const int64_t  in = iz / OH;
+        const int64_t  ioh = iz - in * OH;

-            const int64_t iiw = iow * s0 + ikw * d0 - p0;
-            const int64_t iih = ioh * s1 + ikh * d1 - p1;
+        const int64_t iiw = iow * s0 + ikw * d0 - p0;
+        const int64_t iih = ioh * s1 + ikh * d1 - p1;

-            const int64_t offset_dst =
-                ((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw;
+        const int64_t offset_dst =
+            ((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw;

-            if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
-                dst[offset_dst] = 0.0f;
-            } else {
-                const int64_t offset_src = iic * IC_IH_IW + in * IH_IW;
-                dst[offset_dst] = x[offset_src + iih * IW + iiw];
-            }
+        if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
+            dst[offset_dst] = 0.0f;
+        } else {
+            const int64_t offset_src = iic * IC_IH_IW + in * IH_IW;
+            dst[offset_dst] = x[offset_src + iih * IW + iiw];
        }
    }

@@ -53,7 +51,7 @@ static void im2col_cuda(const float * x, T* dst,
    const int64_t num_blocks = (IC_KH_KW + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;
    const int64_t N_OH = N * OH;
    const int64_t KH_KW = KW*KH;
-    dim3 block_nums(num_blocks, MIN(OW, MAX_GRIDDIM_Y), MIN(N_OH, MAX_GRIDDIM_Z));
+    dim3 block_nums(num_blocks, OW, MIN(N_OH, MAX_GRIDDIM_Z));
    im2col_kernel<<<block_nums, MIN(IC_KH_KW, CUDA_IM2COL_BLOCK_SIZE) , 0, stream>>>(x, dst, IC, IW, IH, OH, OW, KW, KH,
                                                                                     IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW,
                                                                                     s0, s1, p0, p1, d0, d1);
@@ -138,24 +136,23 @@ static  __global__ void im2col_3d_kernel(
    const int64_t ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / KW;
    const int64_t ikw = i % KW;

-    for (int64_t iow = blockIdx.y; iow < OW; iow += MAX_GRIDDIM_Y) {
-        for (int64_t iz = blockIdx.z; iz < N_OD_OH; iz += MAX_GRIDDIM_Z) {
-            const int64_t in  = iz / OD_OH;
-            const int64_t iod = (iz - in*OD_OH) / OH;
-            const int64_t ioh = iz % OH;
+    const int64_t  iow = blockIdx.y;
+    for (int64_t iz = blockIdx.z; iz < N_OD_OH; iz+=MAX_GRIDDIM_Z) {
+        const int64_t in  = iz / OD_OH;
+        const int64_t iod = (iz - in*OD_OH) / OH;
+        const int64_t ioh = iz % OH;

-            const int64_t iiw = iow * s0 + ikw * d0 - p0;
-            const int64_t iih = ioh * s1 + ikh * d1 - p1;
-            const int64_t iid = iod * s2 + ikd * d2 - p2;
+        const int64_t iiw = iow * s0 + ikw * d0 - p0;
+        const int64_t iih = ioh * s1 + ikh * d1 - p1;
+        const int64_t iid = iod * s2 + ikd * d2 - p2;

-            const int64_t offset_dst = in*OD_OH_OW_IC_KD_KH_KW + iod*OH_OW_IC_KD_KH_KW + ioh*OW_IC_KD_KH_KW + iow*IC_KD_KH_KW + iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw;
+        const int64_t offset_dst = in*OD_OH_OW_IC_KD_KH_KW + iod*OH_OW_IC_KD_KH_KW + ioh*OW_IC_KD_KH_KW + iow*IC_KD_KH_KW + iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw;

-            if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) {
-                dst[offset_dst] = 0.0f;
-            } else {
-                const int64_t offset_src = ((in * IC + iic) * stride_q) + (iid * stride_z) + (iih * stride_y) + (iiw * stride_x);
-                dst[offset_dst] = src[offset_src];
-            }
+        if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) {
+            dst[offset_dst] = 0.0f;
+        } else {
+            const int64_t offset_src = ((in * IC + iic) * stride_q) + (iid * stride_z) + (iih * stride_y) + (iiw * stride_x);
+            dst[offset_dst] = src[offset_src];
        }
    }
 }
@@ -181,7 +178,7 @@ static void im2col_3d_cuda(const float * src, T* dst,
    const int64_t OH_OW_IC_KD_KH_KW = OH*OW*IC*KD*KH*KW;
    const int64_t OW_IC_KD_KH_KW = OW*IC*KD*KH*KW;
    const int64_t num_blocks = (IC_KD_KH_KW + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;
-    dim3 block_nums(num_blocks, MIN(OW, MAX_GRIDDIM_Y), MIN(N_OD_OH, MAX_GRIDDIM_Z));
+    dim3 block_nums(num_blocks, OW, MIN(N_OD_OH, MAX_GRIDDIM_Z));
    im2col_3d_kernel<<<block_nums, MIN(IC_KD_KH_KW, CUDA_IM2COL_BLOCK_SIZE) , 0, stream>>>(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW,
                                                                                           OH_OW, KD_KH_KW, ID_IH_IW, KH_KW, IH_IW, IC_ID_IH_IW,
                                                                                           IC_KD_KH_KW, OW_KD_KH_KW, OD_OH_OW_IC_KD_KH_KW,
@@ -80,7 +80,6 @@ namespace ggml_cuda_mma {
        DATA_LAYOUT_J_MAJOR           = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3.
        DATA_LAYOUT_I_MAJOR_MIRRORED  = 20, // Volta, matrix A&B for RDNA3.
        DATA_LAYOUT_J_MAJOR_MIRRORED  = 30,
-        DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (RDNA4/CDNA), convert to float to unscramble.
    };
    // Implemented mma combinations are:
    //   - (I_MAJOR, I_MAJOR)          -> I_MAJOR
@@ -313,19 +312,13 @@ namespace ggml_cuda_mma {
        half2 x[ne] = {{0.0f, 0.0f}};

        static constexpr __device__ bool supported() {
-            if (I == 16 && J ==  8) return true;
-            if (I == 16 && J == 16) return true;
-            if (I == 32 && J ==  8) return true;
+            if (I == 16 && J == 8) return true;
            return false;
        }

        static __device__ __forceinline__ int get_i(const int l) {
            if constexpr (I == 16 && J == 8) {
                return threadIdx.x % 16;
-            } else if constexpr (I == 16 && J == 16) {
-                return threadIdx.x % 16;
-            } else if constexpr (I == 32 && J == 8) {
-                return (threadIdx.x % 16) * 2 + l / (ne/2);
            } else {
                NO_DEVICE_CODE;
                return -1;
@@ -334,15 +327,7 @@ namespace ggml_cuda_mma {

        static __device__ __forceinline__ int get_j(const int l) {
            if constexpr (I == 16 && J == 8) {
-                return (threadIdx.x / 16) * ne + l;
-            } else if constexpr (I == 16 && J == 16) {
-#ifdef RDNA3
-                return l*2 + (threadIdx.x / 16);
-#else
-                return (threadIdx.x / 16) * ne + l;
-#endif // RDNA3
-            } else if constexpr (I == 32 && J == 8) {
-                return (threadIdx.x / 16) * (ne/2) + l % (ne/2);
+                return ne * (threadIdx.x / 16) + l;
            } else {
                NO_DEVICE_CODE;
                return -1;
@@ -353,19 +338,13 @@ namespace ggml_cuda_mma {
        half2 x[ne] = {{0.0f, 0.0f}};

        static constexpr __device__ bool supported() {
-            if (I == 16 && J ==  8) return true;
-            if (I == 16 && J == 16) return true;
-            if (I == 32 && J ==  8) return true;
+            if (I == 16 && J == 8) return true;
            return false;
        }

        static __device__ __forceinline__ int get_i(const int l) {
            if constexpr (I == 16 && J == 8) {
                return threadIdx.x % 16;
-            } else if constexpr (I == 16 && J == 16) {
-                return threadIdx.x % 16;
-            } else if constexpr (I == 32 && J == 8) {
-                return (threadIdx.x % 16) * 2 + l / (ne/2);
            } else {
                NO_DEVICE_CODE;
                return -1;
@@ -374,11 +353,7 @@ namespace ggml_cuda_mma {

        static __device__ __forceinline__ int get_j(const int l) {
            if constexpr (I == 16 && J == 8) {
-                return (threadIdx.x / 16) * ne + l;
-            } else if constexpr (I == 16 && J == 16) {
-                return (threadIdx.x / 16) * ne + l;
-            } else if constexpr (I == 32 && J == 8) {
-                return (threadIdx.x / 16) * (ne/2) + l % (ne/2);
+                return ne * (threadIdx.x / 16) + l;
            } else {
                NO_DEVICE_CODE;
                return -1;
@@ -541,15 +516,12 @@ namespace ggml_cuda_mma {
            if (I == 16 && J == 16) return true;
            if (I == 16 && J == 8)  return true;
            if (I == 16 && J == 4)  return true;
-            if (I == 32 && J == 8)  return true;
            return false;
        }

-        static __device__ __forceinline__ int get_i(const int l) {
-            if constexpr (I == 16) {
+        static __device__ __forceinline__ int get_i(const int /*l*/) {
+            if constexpr (supported()) {
                return threadIdx.x % 16;
-            } else if constexpr (I == 32) {
-                return (threadIdx.x % 16) * 2 + l / (ne/2);
            } else {
                NO_DEVICE_CODE;
                return -1;
@@ -557,10 +529,8 @@ namespace ggml_cuda_mma {
        }

        static __device__ __forceinline__ int get_j(const int l) {
-            if constexpr (I == 16) {
+            if constexpr (supported()) {
                return l;
-            } else if constexpr (I == 32) {
-                return l % (ne/2);
            } else {
                NO_DEVICE_CODE;
                return -1;
@@ -674,40 +644,6 @@ namespace ggml_cuda_mma {
        }
    };

-    template <int I_, int J_>
-    struct tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> {
-        static constexpr int         I  = I_;
-        static constexpr int         J  = J_;
-        static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_SCRAMBLED;
-
-        static constexpr int ne = I * J / ggml_cuda_get_physical_warp_size();
-        half2 x[ne] = {{0.0f, 0.0f}};
-
-        static constexpr __device__ bool supported() {
-            if (I == 16 && J == 16) return true;
-            return false;
-        }
-
-        static __device__ __forceinline__ int get_i(const int l) {
-            return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_i(l);
-        }
-    };
-
-    static __device__ __forceinline__ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> unscramble(const tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & t) {
-#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
-        tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> ret;
-#pragma unroll
-        for (int l0 = 0; l0 < t.ne/2; ++l0) {
-            ret.x[2*l0 + 0] =  __lows2half2(t.x[l0], t.x[l0 + t.ne/2]);
-            ret.x[2*l0 + 1] = __highs2half2(t.x[l0], t.x[l0 + t.ne/2]);
-        }
-        return ret;
-#else
-        NO_DEVICE_CODE;
-        GGML_UNUSED(t);
-#endif // defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
-    }
-
 #if defined(TURING_MMA_AVAILABLE)
    template <int I, int J>
    static __device__ __forceinline__ tile<I, J/2, half2> get_half2(const tile<I, J, float> & tile_float) {
@@ -724,21 +660,6 @@ namespace ggml_cuda_mma {
        ret.x[0] = ggml_cuda_movmatrix(t.x[0]);
        ret.x[1] = ggml_cuda_movmatrix(t.x[1]);

-        return ret;
-    }
-#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
-    static __device__ __forceinline__ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> get_half2(
-            const tile<16, 16, float, DATA_LAYOUT_I_MAJOR> & tile_float) {
-        tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> ret;
-#pragma unroll
-        for (int l = 0; l < tile_float.ne; ++l) {
-            float tmp[2];
-            int i = threadIdx.x / 16;
-            tmp[i] = tile_float.x[l];
-            i ^= 1;
-            tmp[i] = __shfl_xor_sync(0xFFFFFFFF, tile_float.x[l], 16, WARP_SIZE);
-            ret.x[l] = make_half2(tmp[0], tmp[1]);
-        }
        return ret;
    }
 #elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
@@ -881,35 +802,21 @@ namespace ggml_cuda_mma {
 #endif // defined(VOLTA_MMA_AVAILABLE)
    }

-    template <int I, typename T, data_layout dl>
+    template <typename T>
    static __device__ __forceinline__ void load_ldmatrix_trans(
-            tile<I, 8, T, dl> & t, const T * __restrict__ xs0, const int stride) {
+            tile<16, 8, T> & t, const T * __restrict__ xs0, const int stride) {
 #ifdef TURING_MMA_AVAILABLE
-        static_assert(I == 16, "bad tile width");
-        static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");
        int * xi = (int *) t.x;
        const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2);
        asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];"
            : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3])
            : "l"(xs));
 #elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
-        static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");
-        if constexpr (I == 32) {
+        half * xh = (half *) t.x;
 #pragma unroll
-            for (int l0 = 0; l0 < t.ne/2; ++l0) {
-                const half2 tmp0 = xs0[(2*t.get_j(l0) + 0)*stride + t.get_i(l0)/2];
-                const half2 tmp1 = xs0[(2*t.get_j(l0) + 1)*stride + t.get_i(l0)/2];
-
-                t.x[l0]          =  __lows2half2(tmp0, tmp1);
-                t.x[l0 + t.ne/2] = __highs2half2(tmp0, tmp1);
-            }
-        } else {
-            half * xh = (half *) t.x;
-#pragma unroll
-            for (int l = 0; l < t.ne; ++l) {
-                xh[2*l + 0] = ((const half *) xs0)[(2*t.get_j(l) + 0)*(2*stride) + t.get_i(l)];
-                xh[2*l + 1] = ((const half *) xs0)[(2*t.get_j(l) + 1)*(2*stride) + t.get_i(l)];
-            }
+        for (int l = 0; l < t.ne; ++l) {
+            xh[2*l + 0] = ((const half *) xs0)[(2*t.get_j(l) + 0)*(2*stride) + t.get_i(l)];
+            xh[2*l + 1] = ((const half *) xs0)[(2*t.get_j(l) + 1)*(2*stride) + t.get_i(l)];
        }
 #else
        GGML_UNUSED_VARS(t, xs0, stride);
@@ -1065,20 +972,6 @@ namespace ggml_cuda_mma {
 #endif // TURING_MMA_AVAILABLE
    }

-    static __device__ __forceinline__ void mma(
-            tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR> & A,
-            const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR> & B) {
-#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
-        tile<16, 8, half2>       * D16 = (tile<16, 8, half2>       *) &D;
-        const tile<16, 8, half2> * A16 = (const tile<16, 8, half2> *) &A;
-        mma(D16[0], A16[0], B);
-        mma(D16[1], A16[1], B);
-#else
-        GGML_UNUSED_VARS(D, A, B);
-        NO_DEVICE_CODE;
-#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)
-    }
-
    template <data_layout dl_ab, data_layout dl_d>
    static __device__ __forceinline__ void mma(
            tile<16, 8, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<8, 8, float, dl_ab> & B) {
@@ -1403,22 +1296,6 @@ namespace ggml_cuda_mma {
 #endif // defined(VOLTA_MMA_AVAILABLE)
    }

-    static __device__ __forceinline__ void mma(
-            tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> & D, const tile<32,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & A,
-            const tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) {
-#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
-        using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;
-        halfx16_t       * xD = (halfx16_t       *) D.x;
-        const halfx16_t * xA = (const halfx16_t *) A.x;
-        const halfx16_t * xB = (const halfx16_t *) B.x;
-        xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[0], xB[0], xD[0], /*opsel =*/ 0);
-        xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[1], xB[0], xD[0], /*opsel =*/ 1);
-#else
-        GGML_UNUSED_VARS(D, A, B);
-        NO_DEVICE_CODE;
-#endif // TURING_MMA_AVAILABLE
-    }
-
    template <data_layout dl_d, data_layout dl_ab>
    static __device__ __forceinline__ void mma(
            tile<16, 16, int, dl_d> & D, const tile<16, 4, int, dl_ab> & A, const tile<16, 4, int, dl_ab> & B) {
@@ -54,31 +54,15 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
    const int64_t dps2 = ne2 / ne02;
    const int64_t dps3 = ne3 / ne03;

-    if (dps2 == 1 && ne2 > 1) {
-        // src0 has uniform stride s02 along dim 2; batch the inner loop with a strided GEMM
-        GGML_ASSERT(ne2 <= std::numeric_limits<int>::max());
-        const int batch_count = (int) ne2;
-        for (int64_t i3 = 0; i3 < ne3; ++i3) {
+    // TODO batched matrix multiplication
+    for (int64_t i3 = 0; i3 < ne3; ++i3) {
+        for (int64_t i2 = 0; i2 < ne2; ++i2) {
            CUBLAS_CHECK(
-                cublasSgemmStridedBatched(handle, CUBLAS_OP_N, src1_cublas_op,
+                cublasSgemm(handle, CUBLAS_OP_N, src1_cublas_op,
                        ne0, ne1, ne01,
-                        &alpha, src0_d + (i3/dps3)*s03, lda, s02,
-                                src1_d +  i3     *s13, ldb, s12,
-                        &beta,  dst_d  +  i3     *s3,  ldc, s2,
-                        batch_count));
-        }
-    } else {
-        // Fallback: ne2 == 1 (no batching benefit) or dps2 > 1 (src0 broadcast along dim 2
-        // with non-uniform stride; would need cublasSgemmBatched with pointer arrays).
-        for (int64_t i3 = 0; i3 < ne3; ++i3) {
-            for (int64_t i2 = 0; i2 < ne2; ++i2) {
-                CUBLAS_CHECK(
-                    cublasSgemm(handle, CUBLAS_OP_N, src1_cublas_op,
-                            ne0, ne1, ne01,
-                            &alpha, src0_d + (i3/dps3)*s03 + (i2/dps2)*s02, lda,
-                                    src1_d +  i3      *s13 +  i2      *s12, ldb,
-                            &beta,  dst_d  +  i3      *s3  +  i2      *s2,  ldc));
-            }
+                        &alpha, src0_d + (i3/dps3)*s03 + (i2/dps2)*s02, lda,
+                                src1_d +  i3      *s13 +  i2      *s12, ldb,
+                        &beta,  dst_d  +  i3      *s3  +  i2      *s2,  ldc));
        }
    }
 }
@@ -1,72 +0,0 @@
-#include "snake.cuh"
-#include "convert.cuh"
-
-// Fused Snake activation: y = x + sin^2(a * x) * inv_b
-// x: [T, C] (T contiguous), a: [1, C], inv_b: [1, C]
-// Supports F32, F16, BF16 data with F32 compute.
-
-template <typename T>
-static __global__ void snake_kernel(
-        const T     * __restrict__ x,
-        const float * __restrict__ a,
-        const float * __restrict__ inv_b,
-        T           * __restrict__ dst,
-        const int    total,
-        const uint3  T_len_fastdiv) {
-    const int idx = blockIdx.x * blockDim.x + threadIdx.x;
-    if (idx >= total) return;
-
-    const int c = (int) fastdiv((uint32_t) idx, T_len_fastdiv);
-
-    const float xi = ggml_cuda_cast<float>(x[idx]);
-    const float s  = sinf(a[c] * xi);
-    dst[idx] = ggml_cuda_cast<T>(xi + s * s * inv_b[c]);
-}
-
-// Internal launcher with explicit x/a/inv_b/dst tensors.
-// Shared by the public op (reads dst->src) and the fusion path (explicit args).
-static void launch_snake(ggml_backend_cuda_context & ctx,
-                         const ggml_tensor * x,
-                         const ggml_tensor * a,
-                         const ggml_tensor * inv_b,
-                         ggml_tensor *       dst) {
-    const float * a_d     = (const float *)a->data;
-    const float * inv_b_d = (const float *)inv_b->data;
-
-    const int   T = (int)x->ne[0];
-    const int   C = (int)x->ne[1];
-    const int   total = T * C;
-    const uint3 T_len_fastdiv = init_fastdiv_values((uint64_t) T);
-
-    const int block_size = 256;
-    const int grid_size  = (total + block_size - 1) / block_size;
-
-    cudaStream_t stream = ctx.stream();
-
-    switch (x->type) {
-        case GGML_TYPE_F32: {
-            snake_kernel<<<grid_size, block_size, 0, stream>>>(
-                (const float *)x->data, a_d, inv_b_d, (float *)dst->data, total, T_len_fastdiv);
-        } break;
-        case GGML_TYPE_F16: {
-            snake_kernel<<<grid_size, block_size, 0, stream>>>(
-                (const half *)x->data, a_d, inv_b_d, (half *)dst->data, total, T_len_fastdiv);
-        } break;
-        case GGML_TYPE_BF16: {
-            snake_kernel<<<grid_size, block_size, 0, stream>>>(
-                (const nv_bfloat16 *)x->data, a_d, inv_b_d, (nv_bfloat16 *)dst->data, total, T_len_fastdiv);
-        } break;
-        default:
-            GGML_ABORT("snake: unsupported type");
-    }
-}
-
-// Fusion entry: caller supplies x/a/inv_b explicitly from the matched
-// mul -> sin -> sqr -> mul -> add pattern. The dst is the trailing add output.
-void ggml_cuda_op_snake_fused(ggml_backend_cuda_context & ctx,
-                              const ggml_tensor * x,
-                              const ggml_tensor * a,
-                              const ggml_tensor * inv_b,
-                              ggml_tensor *       dst) {
-    launch_snake(ctx, x, a, inv_b, dst);
-}
@@ -1,8 +0,0 @@
-#include "common.cuh"
-
-// Fusion entry point. Caller supplies x/a/inv_b explicitly.
-void ggml_cuda_op_snake_fused(ggml_backend_cuda_context & ctx,
-                              const ggml_tensor * x,
-                              const ggml_tensor * a,
-                              const ggml_tensor * inv_b,
-                              ggml_tensor *       dst);
@@ -2,5 +2,4 @@

 #include "../fattn-mma-f16.cuh"

-DECL_FATTN_MMA_F16_CASE(192, 128, 1, 16);
 DECL_FATTN_MMA_F16_CASE(576, 512, 1, 16);
@@ -7,6 +7,5 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 1, 8);
 DECL_FATTN_MMA_F16_CASE(96, 96, 1, 8);
 DECL_FATTN_MMA_F16_CASE(112, 112, 1, 8);
 DECL_FATTN_MMA_F16_CASE(128, 128, 1, 8);
-DECL_FATTN_MMA_F16_CASE(192, 128, 1, 8);
 DECL_FATTN_MMA_F16_CASE(256, 256, 1, 8);
 DECL_FATTN_MMA_F16_CASE(512, 512, 1, 8);
@@ -2,5 +2,4 @@

 #include "../fattn-mma-f16.cuh"

-DECL_FATTN_MMA_F16_CASE(192, 128, 2, 16);
 DECL_FATTN_MMA_F16_CASE(576, 512, 2, 16);
@@ -7,6 +7,5 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 2, 8);
 DECL_FATTN_MMA_F16_CASE(96, 96, 2, 8);
 DECL_FATTN_MMA_F16_CASE(112, 112, 2, 8);
 DECL_FATTN_MMA_F16_CASE(128, 128, 2, 8);
-DECL_FATTN_MMA_F16_CASE(192, 128, 2, 8);
 DECL_FATTN_MMA_F16_CASE(256, 256, 2, 8);
 DECL_FATTN_MMA_F16_CASE(512, 512, 2, 8);
@@ -2,5 +2,4 @@

 #include "../fattn-mma-f16.cuh"

-DECL_FATTN_MMA_F16_CASE(192, 128, 4, 16);
 DECL_FATTN_MMA_F16_CASE(576, 512, 4, 16);
@@ -7,6 +7,5 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 4, 8);
 DECL_FATTN_MMA_F16_CASE(96, 96, 4, 8);
 DECL_FATTN_MMA_F16_CASE(112, 112, 4, 8);
 DECL_FATTN_MMA_F16_CASE(128, 128, 4, 8);
-DECL_FATTN_MMA_F16_CASE(192, 128, 4, 8);
 DECL_FATTN_MMA_F16_CASE(256, 256, 4, 8);
 DECL_FATTN_MMA_F16_CASE(512, 512, 4, 8);
@@ -7,6 +7,5 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 8, 8);
 DECL_FATTN_MMA_F16_CASE(96, 96, 8, 8);
 DECL_FATTN_MMA_F16_CASE(112, 112, 8, 8);
 DECL_FATTN_MMA_F16_CASE(128, 128, 8, 8);
-DECL_FATTN_MMA_F16_CASE(192, 128, 8, 8);
 DECL_FATTN_MMA_F16_CASE(256, 256, 8, 8);
 DECL_FATTN_MMA_F16_CASE(512, 512, 8, 8);
@@ -1,5 +0,0 @@
-// This file has been autogenerated by generate_cu_files.py, do not edit manually.
-
-#include "../fattn-tile.cuh"
-
-DECL_FATTN_TILE_CASE(192, 128);
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Georgi Gerganov	efa2f8e5a7	naming : improve consistency	2026-05-08 12:24:57 +03:00
Georgi Gerganov	778f9e247e	tools : update readme	2026-05-08 11:55:16 +03:00
Georgi Gerganov	1dbc054da5	server : fix slot ctx_drft ptr	2026-05-08 11:55:05 +03:00
Georgi Gerganov	161eae0adf	spec : fix n_past type	2026-05-08 11:54:32 +03:00
Georgi Gerganov	e5b1401318	speculative-simple : update	2026-05-08 11:09:34 +03:00
Georgi Gerganov	3b1a8df8fd	server : clean-up + dry	2026-05-08 10:20:01 +03:00
Georgi Gerganov	233d1aee69	server : add comment [no ci]	2026-05-08 08:50:23 +03:00
Georgi Gerganov	12c7cfbe83	server : fix URL for draft model	2026-05-08 08:03:49 +03:00
Georgi Gerganov	6a4b05a030	server : fix mtmd draft processing	2026-05-08 08:02:11 +03:00
Georgi Gerganov	8be14e40de	spec : handle draft running out of context	2026-05-08 07:11:51 +03:00
Georgi Gerganov	7e118cdce0	cont : process images throught the draft context	2026-05-07 21:44:09 +03:00
Georgi Gerganov	ae6703fa89	cont : pass correct n_past for drafting	2026-05-07 21:44:08 +03:00
Georgi Gerganov	0239f4c611	cont : handle non-ckpt models	2026-05-07 21:44:08 +03:00
Georgi Gerganov	c7facb0fe1	cont : async drft eval when possible	2026-05-07 21:44:08 +03:00
Georgi Gerganov	08c8012bde	cont : sync main and drft contexts	2026-05-07 21:44:08 +03:00
Georgi Gerganov	de35b1255c	server, spec : transition to unified spec context	2026-05-07 21:44:08 +03:00
Georgi Gerganov	1afee5b262	server : improve ctx names [no ci]	2026-05-07 21:44:08 +03:00
Georgi Gerganov	11fd5e7272	server : draft prompt cache and checkpoints [no ci]	2026-05-07 21:44:08 +03:00
Georgi Gerganov	c97dc3605e	server : sketch the ctx_dft decode loop [no ci]	2026-05-07 21:44:08 +03:00
Georgi Gerganov	8a50f6f0b9	cont : dedup ctx_seq_rm_type [no ci]	2026-05-07 21:44:07 +03:00
Georgi Gerganov	77269ad8a7	cont : pass seq_id [no ci]	2026-05-07 21:44:07 +03:00
Georgi Gerganov	4550f0f08b	spec : update common_speculative_init() [no ci]	2026-05-07 21:44:07 +03:00
Georgi Gerganov	befc7ef635	spec : drop support for incompatible vocabs [no ci]	2026-05-07 21:44:07 +03:00
Georgi Gerganov	2c9a40849f	spec : refactor [no ci]	2026-05-07 21:44:07 +03:00