mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-28 19:17:47 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
b387ddfd84 | ||
|
|
a43c3986b4 | ||
|
|
90c26fcd4b | ||
|
Georgi GerganovandGitHub
|
8663224818 | ||
|
|
f5e85d43a0 | ||
|
|
511f9c1379 | ||
|
Xuan-Son NguyenandGitHub
|
b19cbe925b |
@@ -1,12 +1,12 @@
|
||||
ARG OPENVINO_VERSION_MAJOR=2026.3
|
||||
ARG OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c
|
||||
ARG OPENVINO_VERSION_MAJOR=2026.3.1
|
||||
ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
|
||||
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
|
||||
ARG IGC_VERSION=v2.38.2
|
||||
ARG IGC_VERSION_FULL=2_2.38.2+22051
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.27.39122.11
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.27.39122.11-0
|
||||
ARG IGC_VERSION=v2.40.13
|
||||
ARG IGC_VERSION_FULL=2_2.40.13+22418
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.31.39395.13
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
|
||||
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
|
||||
|
||||
@@ -41,8 +41,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
||||
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -69,8 +69,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
||||
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -32,6 +32,8 @@ env:
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
# TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback`
|
||||
CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-rollback"
|
||||
|
||||
jobs:
|
||||
ubuntu-24-openvino:
|
||||
@@ -39,8 +41,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
||||
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -78,26 +80,24 @@ jobs:
|
||||
|
||||
- name: Test (CPU)
|
||||
id: cmake_test_cpu
|
||||
# TODO: fix and re-enable the `test-llama-archs` test below
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 2000
|
||||
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
||||
|
||||
- name: Test (GPU)
|
||||
id: cmake_test_gpu
|
||||
# TODO: fix and re-enable the `test-llama-archs` test below
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 3000
|
||||
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
||||
|
||||
openvino-windows-2022:
|
||||
runs-on: windows-2022
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
||||
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -159,14 +159,13 @@ jobs:
|
||||
- name: Test (CPU)
|
||||
id: cmake_test_cpu
|
||||
shell: cmd
|
||||
# TODO: fix and re-enable the `test-llama-archs` test below
|
||||
run: |
|
||||
REM Find extracted OpenVINO folder dynamically
|
||||
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
|
||||
call "%OPENVINO_ROOT%\setupvars.bat"
|
||||
|
||||
cd build
|
||||
ctest --test-dir ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" -C Release --verbose --timeout 3000
|
||||
ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
|
||||
@@ -288,8 +288,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
||||
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -415,8 +415,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
||||
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
||||
|
||||
steps:
|
||||
- name: Set OpenVINO version output
|
||||
@@ -529,8 +529,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
||||
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
||||
|
||||
steps:
|
||||
- name: Set OpenVINO version output
|
||||
|
||||
@@ -189,8 +189,8 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
|
||||
fi
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
|
||||
|
||||
# TODO: fix and re-enable the `test-llama-archs` test below
|
||||
CTEST_EXTRA="-E test-llama-archs|test-recurrent-state-rollback-nemotron-h"
|
||||
# TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback*`
|
||||
CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-rollback"
|
||||
fi
|
||||
|
||||
## helpers
|
||||
|
||||
+46
-40
@@ -22,8 +22,8 @@ The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a t
|
||||
- [0. Prerequisites](#0-prerequisites)
|
||||
- [1. Install OpenVINO Runtime](#1-install-openvino-runtime)
|
||||
- [2. Build llama.cpp with OpenVINO Backend](#2-build-llamacpp-with-openvino-backend)
|
||||
- [Automated Ubuntu Build Script](#automated-ubuntu-build-script)
|
||||
- [Automated Windows Build Script](#automated-windows-build-script)
|
||||
- [Ubuntu Build Script](#ubuntu-build-script)
|
||||
- [Windows Build Script](#windows-build-script)
|
||||
- [3. Download Sample Model](#3-download-sample-model)
|
||||
- [4. Run Inference with OpenVINO Backend](#4-run-inference-with-openvino-backend)
|
||||
- [5. Docker Build](#5-docker-build)
|
||||
@@ -96,7 +96,7 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
|
||||
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
|
||||
- Note: The NPU operates in stateless mode only.
|
||||
- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.18.38308.1 | Intel NPU Driver 1.33.0.
|
||||
- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.35.0.
|
||||
- See [Known Limitations](#known-limitations) for context on observed failures.
|
||||
|
||||
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
|
||||
@@ -105,27 +105,32 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| | | | |
|
||||
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
|
||||
| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
|
||||
| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
|
||||
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
|
||||
| | | | |
|
||||
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
|
||||
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
|
||||
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
|
||||
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| | | | |
|
||||
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
|
||||
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
@@ -133,10 +138,10 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| | | | |
|
||||
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
|
||||
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| | | | |
|
||||
| [gpustack/bge-m3-Q4_K_M.gguf](https://huggingface.co/gpustack/bge-m3-GGUF) | ✓ | ✗ | ✗ |
|
||||
|
||||
@@ -217,18 +222,18 @@ cmake --build build\ReleaseOV --parallel
|
||||
> [!NOTE]
|
||||
> The Windows install path is `C:\Intel\openvino` (no spaces) to avoid quoting problems some CMake/Ninja toolchains have with `C:\Program Files (x86)\...`. Adjust to wherever you installed OpenVINO Runtime. From `cmd`, run `C:\Intel\openvino\setupvars.bat`; from PowerShell, run `& "C:\Intel\openvino\setupvars.ps1"` instead. Once the build is finished you can launch the binaries from any `cmd` or `PowerShell` window after sourcing the matching `setupvars` script for that shell.
|
||||
|
||||
#### Automated Ubuntu Build Script
|
||||
#### Ubuntu Build Script
|
||||
|
||||
For Ubuntu24 users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
|
||||
Save the following as `ubuntu-llamacpp-ov-install.sh` next to where you want the `llama.cpp` folder to land, then run it:
|
||||
Save the following as `build-llamacpp-ov.sh` next to where you want the `llama.cpp` folder to land, then run it:
|
||||
|
||||
```bash
|
||||
chmod +x ubuntu-llamacpp-ov-install.sh
|
||||
./ubuntu-llamacpp-ov-install.sh
|
||||
chmod +x build-llamacpp-ov.sh
|
||||
./build-llamacpp-ov.sh
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary>Click to expand <code>ubuntu-llamacpp-ov-install.sh</code></summary>
|
||||
<summary>Click to expand <code>build-llamacpp-ov.sh</code></summary>
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
@@ -237,8 +242,8 @@ chmod +x ubuntu-llamacpp-ov-install.sh
|
||||
# ============================================
|
||||
set -euo pipefail
|
||||
|
||||
OPENVINO_VERSION_MAJOR="2026.3"
|
||||
OPENVINO_VERSION_FULL="2026.3.0.22451.bd8d6542e3c"
|
||||
OPENVINO_VERSION_MAJOR="2026.3.1"
|
||||
OPENVINO_VERSION_FULL="2026.3.1.22476.56d9685302d"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}"
|
||||
@@ -313,8 +318,9 @@ fi
|
||||
echo "============================================"
|
||||
echo "Configuring with CMake..."
|
||||
echo "============================================"
|
||||
# shellcheck disable=SC1091
|
||||
set +u
|
||||
source "${OPENVINO_ROOT}/setupvars.sh"
|
||||
set -u
|
||||
|
||||
cmake -B build/ReleaseOV -G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
@@ -334,27 +340,27 @@ echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf"
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> The script pins OpenVINO `2026.3` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
|
||||
> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
|
||||
|
||||
</details>
|
||||
|
||||
#### Automated Windows Build Script
|
||||
#### Windows Build Script
|
||||
|
||||
For Windows users, the following `.bat` script automates the prerequisite installs (Git, Ninja, CMake, Visual Studio 2022 Build Tools, vcpkg + OpenCL), the OpenVINO Runtime download/extract, and the Ninja-based llama.cpp build.
|
||||
Save the following as `windows-llamacpp-ov-install.bat` next to where you want the `llama.cpp` to land, then run it from either **Command Prompt** or **PowerShell**:
|
||||
Save the following as `build-llamacpp-ov.bat` next to where you want the `llama.cpp` to land, then run it from either **Command Prompt** or **PowerShell**:
|
||||
|
||||
```cmd
|
||||
:: Command Prompt
|
||||
windows-llamacpp-ov-install.bat
|
||||
build-llamacpp-ov.bat
|
||||
```
|
||||
|
||||
```powershell
|
||||
# PowerShell
|
||||
.\windows-llamacpp-ov-install.bat
|
||||
.\build-llamacpp-ov.bat
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary>Click to expand <code>windows-llamacpp-ov-install.bat</code></summary>
|
||||
<summary>Click to expand <code>build-llamacpp-ov.bat</code></summary>
|
||||
|
||||
```bat
|
||||
@echo off
|
||||
@@ -364,8 +370,8 @@ REM ============================================
|
||||
REM llama.cpp OpenVINO Build Script (Ninja)
|
||||
REM ============================================
|
||||
|
||||
set "OPENVINO_VERSION_MAJOR=2026.3"
|
||||
set "OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c"
|
||||
set "OPENVINO_VERSION_MAJOR=2026.3.1"
|
||||
set "OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d"
|
||||
|
||||
set "SCRIPT_DIR=%~dp0"
|
||||
set "VCPKG_DIR=C:\vcpkg"
|
||||
@@ -453,9 +459,6 @@ if exist "%OPENVINO_INSTALL_DIR%\setupvars.bat" (
|
||||
)
|
||||
|
||||
REM Move the single top-level folder contents into the versioned install dir.
|
||||
REM NOTE: delayed expansion (!VAR!) is required because the surrounding else( ... )
|
||||
REM block is parsed once up-front, so %OPENVINO_EXTRACTED% would expand to "" here
|
||||
REM and xcopy would then treat "\*" as C:\* and fail with "Cannot perform a cyclic copy".
|
||||
set "OPENVINO_EXTRACTED="
|
||||
for /d %%i in ("%OPENVINO_EXTRACT_TMP%\*") do set "OPENVINO_EXTRACTED=%%i"
|
||||
if not defined OPENVINO_EXTRACTED (
|
||||
@@ -547,7 +550,7 @@ endlocal
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> The script pins OpenVINO `2026.3` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
|
||||
> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
|
||||
|
||||
</details>
|
||||
|
||||
@@ -712,6 +715,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `
|
||||
| `GGML_OPENVINO_CACHE_DIR` | String | `not set` | Directory for OpenVINO model caching (recommended: `/tmp/ov_cache`). Enables model caching when set. **Not supported on NPU devices.** |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | String | `not set` | Directory for the frontend compiled-model cache. When set, OpenVINO compiled models are exported as blobs and imported on later runs to skip weight requantization, graph conversion, and compilation for matching single-graph models. |
|
||||
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. |
|
||||
| `GGML_OPENVINO_NPU_COMPILE_CONFIG` | String | `not set` | NPU-only compiler mode parameters forwarded to OpenVINO as `NPU_COMPILATION_MODE_PARAMS`, for example `optimization-level=3`. |
|
||||
| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Enable stateful KV cache for better performance. Recommended on CPU, GPU. |
|
||||
| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
|
||||
@@ -725,9 +729,11 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `
|
||||
| `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. |
|
||||
| `GGML_OPENVINO_DEBUG_OUTPUT` | Boolean | `0` | Enable output debugging and print output tensor info. |
|
||||
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | Boolean | `0` | Print tensor address map once. |
|
||||
| `GGML_OPENVINO_LOG_UNSUPPORTED_OPS`| Boolean | `0` | Log warning messages with tensor details and rejection reasons for any ops not supported by the OpenVINO backend. Emits at `WARN` level (requires `--log-verbosity >= 2`, enabled by default). |
|
||||
|
||||
> [!NOTE]
|
||||
>`GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature to allow stateful execution for managing the KV cache internally inside the OpenVINO model, improving performance on CPUs and GPUs. Stateful execution is not effective on NPUs, and not all models currently support this feature. This feature is experimental and has been validated only with the llama-simple, llama-cli, llama-bench, and llama-run applications and is recommended to enable for the best performance. Other applications, such as llama-server and llama-perplexity, are not yet supported.
|
||||
> - `GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature to allow stateful execution for managing the KV cache internally inside the OpenVINO model, improving performance on CPUs and GPUs. Stateful execution is not effective on NPUs, and not all models currently support this feature. This feature is experimental and has been validated only with the llama-simple, llama-cli, llama-bench, and llama-run applications and is recommended to enable for the best performance. Other applications, such as llama-server and llama-perplexity, are not yet supported.
|
||||
> - `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` emits logs at `WARN` level (`GGML_LOG_WARN`), which requires application log verbosity `--log-verbosity >= 2` (or `-lv 2`).
|
||||
|
||||
### Example Usage
|
||||
|
||||
|
||||
@@ -2936,12 +2936,13 @@ struct ggml_cplan ggml_graph_plan(
|
||||
const int64_t ne10 = node->src[1]->ne[0]; // W
|
||||
const int64_t ne11 = node->src[1]->ne[1]; // H
|
||||
const int64_t ne12 = node->src[1]->ne[2]; // Channels In
|
||||
const int64_t ne13 = node->src[1]->ne[3]; // Batch
|
||||
|
||||
GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32);
|
||||
|
||||
cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03;
|
||||
cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12;
|
||||
cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13;
|
||||
|
||||
} break;
|
||||
case GGML_OP_TOP_K:
|
||||
|
||||
+32
-26
@@ -7267,18 +7267,21 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
|
||||
}
|
||||
}
|
||||
|
||||
// permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)
|
||||
// permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches
|
||||
{
|
||||
kernel_t * const wdata = (kernel_t *) params->wdata + nk;
|
||||
for (int i12 = 0; i12 < ne12; i12++) {
|
||||
for (int i11 = 0; i11 < ne11; i11++) {
|
||||
const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);
|
||||
kernel_t * dst_data = wdata + i11*ne10*ne12;
|
||||
for (int i10 = 0; i10 < ne10; i10++) {
|
||||
if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
|
||||
dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);
|
||||
} else {
|
||||
dst_data[i10*ne12 + i12] = src[i10];
|
||||
for (int i13 = 0; i13 < ne13; i13++) {
|
||||
kernel_t * const wdata_b = wdata + i13*ne10*ne11*ne12;
|
||||
for (int i12 = 0; i12 < ne12; i12++) {
|
||||
for (int i11 = 0; i11 < ne11; i11++) {
|
||||
const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
|
||||
kernel_t * dst_data = wdata_b + i11*ne10*ne12;
|
||||
for (int i10 = 0; i10 < ne10; i10++) {
|
||||
if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
|
||||
dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);
|
||||
} else {
|
||||
dst_data[i10*ne12 + i12] = src[i10];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7305,24 +7308,27 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
|
||||
kernel_t * const wdata_src = wdata + nk;
|
||||
|
||||
for (int i2 = ip0; i2 < ip1; i2++) { // Cout
|
||||
float * dst_data = (float *)((char *) dst->data + i2*nb2);
|
||||
kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;
|
||||
for (int i11 = 0; i11 < ne11; i11++) {
|
||||
for (int i10 = 0; i10 < ne10; i10++) {
|
||||
const int i1n = i11*ne10*ne12 + i10*ne12;
|
||||
for (int i01 = 0; i01 < ne01; i01++) {
|
||||
for (int i00 = 0; i00 < ne00; i00++) {
|
||||
float v = 0;
|
||||
if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
|
||||
ggml_vec_dot_f16(ne03, &v, 0,
|
||||
wdata_src + i1n, 0,
|
||||
wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
|
||||
} else {
|
||||
ggml_vec_dot_f32(ne03, &v, 0,
|
||||
wdata_src + i1n, 0,
|
||||
wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
|
||||
for (int i3 = 0; i3 < ne3; i3++) { // batch
|
||||
float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2);
|
||||
kernel_t * wdata_src_b = wdata_src + i3*ne10*ne11*ne12;
|
||||
for (int i11 = 0; i11 < ne11; i11++) {
|
||||
for (int i10 = 0; i10 < ne10; i10++) {
|
||||
const int i1n = i11*ne10*ne12 + i10*ne12;
|
||||
for (int i01 = 0; i01 < ne01; i01++) {
|
||||
for (int i00 = 0; i00 < ne00; i00++) {
|
||||
float v = 0;
|
||||
if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
|
||||
ggml_vec_dot_f16(ne03, &v, 0,
|
||||
wdata_src_b + i1n, 0,
|
||||
wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
|
||||
} else {
|
||||
ggml_vec_dot_f32(ne03, &v, 0,
|
||||
wdata_src_b + i1n, 0,
|
||||
wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
|
||||
}
|
||||
dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
|
||||
}
|
||||
dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -660,6 +660,7 @@ typedef struct {
|
||||
uint64_t nb0;
|
||||
uint64_t nb1;
|
||||
uint64_t nb2;
|
||||
uint64_t nb3;
|
||||
} ggml_metal_kargs_conv_transpose_2d;
|
||||
|
||||
typedef struct {
|
||||
|
||||
@@ -4645,6 +4645,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
|
||||
const int32_t OW = op->ne[0];
|
||||
const int32_t OH = op->ne[1];
|
||||
const int32_t OC = op->ne[2];
|
||||
const int32_t N = op->src[1]->ne[3];
|
||||
|
||||
ggml_metal_kargs_conv_transpose_2d args = {
|
||||
/*.IC =*/ IC,
|
||||
@@ -4657,6 +4658,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
|
||||
/*.nb0 =*/ nb0,
|
||||
/*.nb1 =*/ nb1,
|
||||
/*.nb2 =*/ nb2,
|
||||
/*.nb3 =*/ nb3,
|
||||
};
|
||||
|
||||
auto pipeline = ggml_metal_library_get_pipeline_conv_transpose_2d(lib, op);
|
||||
@@ -4671,7 +4673,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
|
||||
const size_t smem = GGML_PAD(KW * KH * sizeof(float), 16);
|
||||
ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC, KW, KH, 1);
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC * N, KW, KH, 1);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -542,6 +542,232 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
|
||||
{ { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, 3, 0 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, 3, 0 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 3 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 1 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
|
||||
|
||||
@@ -366,7 +366,8 @@ kernel void kernel_conv_transpose_2d(
|
||||
|
||||
const int64_t out_x = tgpig[0];
|
||||
const int64_t out_y = tgpig[1];
|
||||
const int64_t out_c = tgpig[2];
|
||||
const int64_t batch = tgpig[2] / args.OC;
|
||||
const int64_t out_c = tgpig[2] % args.OC;
|
||||
|
||||
const int64_t kw = tpitg[0];
|
||||
const int64_t kh = tpitg[1];
|
||||
@@ -390,7 +391,7 @@ kernel void kernel_conv_transpose_2d(
|
||||
|
||||
if (in_x >= args.IW) continue;
|
||||
|
||||
const int64_t input_idx = (args.IW * args.IH) * in_c + (args.IW) * in_y + in_x;
|
||||
const int64_t input_idx = (args.IW * args.IH) * (args.IC * batch + in_c) + (args.IW) * in_y + in_x;
|
||||
const int64_t kernel_idx = (args.KH * args.KW * args.OC) * in_c + (args.KH * args.KW) * out_c + (args.KW) * kh + kw;
|
||||
|
||||
v += (float)src0[kernel_idx] * src1[input_idx];
|
||||
@@ -408,7 +409,7 @@ kernel void kernel_conv_transpose_2d(
|
||||
total += shared_sum[i];
|
||||
}
|
||||
|
||||
device float * dst_ptr = (device float *) (dst + out_x*args.nb0 + out_y * args.nb1 + out_c*args.nb2);
|
||||
device float * dst_ptr = (device float *) (dst + batch*args.nb3 + out_c*args.nb2 + out_y * args.nb1 + out_x*args.nb0);
|
||||
dst_ptr[0] = total;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
find_package(OpenVINO REQUIRED COMPONENTS Runtime Threading)
|
||||
find_package(OpenCL REQUIRED)
|
||||
|
||||
message(STATUS "Found OpenVINO: ${OpenVINO_DIR} (found version \"${OpenVINO_VERSION}\")")
|
||||
|
||||
file(GLOB_RECURSE GGML_HEADERS_OPENVINO "*.h" "*.hpp")
|
||||
file(GLOB_RECURSE GGML_SOURCES_OPENVINO "*.cpp")
|
||||
|
||||
|
||||
@@ -357,6 +357,18 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
|
||||
break;
|
||||
}
|
||||
case GGML_OP_VIEW: {
|
||||
if (m_is_static && node->src[0] != nullptr &&
|
||||
(node->src[0]->op == GGML_OP_GATED_DELTA_NET || node->src[0]->op == GGML_OP_CONCAT)) {
|
||||
// VIEW slicing a GATED_DELTA_NET combined [attn|state] output, or the conv_input
|
||||
// CONCAT. The consuming CPY/RMS_NORM op recovers the true window at runtime via
|
||||
// ssm_state_size / the fixed conv kernel width, so this VIEW must stay an identity
|
||||
// pass-through of the full source here too (it already is on the dynamic path);
|
||||
// otherwise the generic static-mode Slice below would bake in the *captured*
|
||||
// cgraph's token count, which is wrong once the compiled static model runs with a
|
||||
// different token count (prefill chunk size or 1).
|
||||
op_case = 1;
|
||||
break;
|
||||
}
|
||||
if (node->src[0]->op == GGML_OP_VIEW) {
|
||||
auto * src = node->src[0];
|
||||
if (ggml_nelements(node) != ggml_nelements(src)) {
|
||||
@@ -408,6 +420,23 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_POOL_2D: {
|
||||
const ggml_op_pool pool_mode = static_cast<ggml_op_pool>(node->op_params[0]);
|
||||
switch (pool_mode) {
|
||||
case GGML_OP_POOL_MAX: {
|
||||
op_case = 1;
|
||||
break;
|
||||
}
|
||||
case GGML_OP_POOL_AVG: {
|
||||
op_case = 2;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
op_case = 0;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_CPY: {
|
||||
if (node->src[0]->op == GGML_OP_VIEW) {
|
||||
if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
|
||||
@@ -425,6 +454,31 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
|
||||
is_kvcache(node->src[1]->view_src, nullptr)) {
|
||||
// s_copy defrag remainder writeback: gathered extra state rows copied back into the cache
|
||||
op_case = 3;
|
||||
} else if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) {
|
||||
// op_case 5: KV write for decoder self-attention (dynamic write offset)
|
||||
// op_case 6: KV write for encoder self-attn or cross-attn (static offset)
|
||||
const ggml_tensor * kv_buf = node->src[1]->view_src;
|
||||
if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) {
|
||||
op_case = 6;
|
||||
// Forward-scan the graph for a FLASH_ATTN_EXT that reads from
|
||||
// the same buffer. Having a mask (src[3] != nullptr) implies
|
||||
// decoder self-attention and the write offset is dynamic.
|
||||
for (int i = 0; i < m_cgraph->n_nodes; i++) {
|
||||
const ggml_tensor * n = m_cgraph->nodes[i];
|
||||
if (n->op != GGML_OP_FLASH_ATTN_EXT) {
|
||||
continue;
|
||||
}
|
||||
// K (src[1]) and V (src[2]) are 3-D views whose view_src is
|
||||
// the flat KV buffer we are writing to.
|
||||
if ((n->src[1] != nullptr && n->src[1]->view_src == kv_buf) ||
|
||||
(n->src[2] != nullptr && n->src[2]->view_src == kv_buf)) {
|
||||
if (n->src[3] != nullptr) {
|
||||
op_case = 5; // decoder self-attention: mask present
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -448,6 +502,15 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_FLASH_ATTN_EXT: {
|
||||
if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) {
|
||||
const ggml_tensor * kv_buf = node->src[1]->view_src;
|
||||
if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) {
|
||||
op_case = (node->src[3] != nullptr) ? 1 : 2;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -479,23 +542,35 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
|
||||
|
||||
switch (node->op) {
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
if (node->src[0] == nullptr || node->src[1] == nullptr || node->src[3] == nullptr) {
|
||||
if (node->src[0] == nullptr || node->src[1] == nullptr) {
|
||||
return -1;
|
||||
}
|
||||
switch (node->src[1]->op) {
|
||||
case GGML_OP_PERMUTE:
|
||||
// case 0: node op is FLASH_ATTN_EXT, src 1 not null & op is PERMUTE & the permuted tensor src is the view of cache k
|
||||
if (node->src[1]->src[0] != nullptr && node->src[1]->src[0]->op == GGML_OP_VIEW) {
|
||||
// case 0: src[1] is PERMUTE of a cache VIEW, mask required
|
||||
if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr &&
|
||||
node->src[1]->src[0]->op == GGML_OP_VIEW) {
|
||||
return 0;
|
||||
}
|
||||
break;
|
||||
case GGML_OP_CPY:
|
||||
// case 1: node op is FLASH_ATTN_EXT, src 1 not null & op is CPY & the copied tensor src is PERMUTE & the permuted tensor src is the view of cache k
|
||||
if (node->src[1]->src[0] != nullptr && node->src[1]->src[0]->op == GGML_OP_PERMUTE &&
|
||||
node->src[1]->src[0]->src[0] != nullptr && node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) {
|
||||
// case 1: src[1] is CPY of a PERMUTE(VIEW), mask required
|
||||
if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr &&
|
||||
node->src[1]->src[0]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->src[0] != nullptr &&
|
||||
node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) {
|
||||
return 1;
|
||||
}
|
||||
break;
|
||||
case GGML_OP_VIEW:
|
||||
// cases 4/5/6: whisper - K is a direct non-contiguous VIEW_3D of a KV cache
|
||||
if (node->src[1]->view_src != nullptr) {
|
||||
if (node->src[3] != nullptr) {
|
||||
return 4; // decoder self-attention
|
||||
} else {
|
||||
return 5; // cross-attention or encoder self-attention
|
||||
};
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -548,6 +623,18 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
|
||||
cache_k_permute = node->src[0]->src[0]->src[0];
|
||||
mask = node->src[1];
|
||||
break;
|
||||
case 4:
|
||||
case 5: {
|
||||
// whisper: K is a direct VIEW_3D of the KV buffer, no PERMUTE node
|
||||
auto * cache_k_view = node->src[1]; // VIEW_3D of kv_self.k or kv_cross.k`
|
||||
compute_params.token_len_per_seq = node->src[0]->ne[1];
|
||||
if (attention_pattern_case == 4) {
|
||||
compute_params.attention_size = cache_k_view->ne[1];
|
||||
} else {
|
||||
compute_params.attention_size_static = cache_k_view->ne[1];
|
||||
}
|
||||
continue;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -654,10 +741,8 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
|
||||
ComputeParams::RsWriteback writeback;
|
||||
writeback.slot_begin = (int) (dest_view->view_offs / row_bytes);
|
||||
if (is_conv) {
|
||||
// conv_input column the copied window starts at
|
||||
writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[0]);
|
||||
} else if (is_gdn) {
|
||||
// first row of the state part of the gated-delta-net output
|
||||
writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[1]);
|
||||
}
|
||||
compute_params.rs_writebacks[get_tensor_ov_name(cgraph, node)] = writeback;
|
||||
@@ -718,11 +803,15 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
|
||||
} else if (is_kvcache(input, op)) {
|
||||
// kvcache
|
||||
input_shape = ov::PartialShape{get_shape(input)};
|
||||
if (!m_is_static) {
|
||||
// Whisper.cpp uses a fixed size 1D KV buffer [N, 1, 1, 1] (GGML) or [1, 1, 1, N] (OV).
|
||||
// the token fill level is handled by token_len_per_seq + dynamic mask input.
|
||||
// skip dynamic dim and stateful reshape for this layout.
|
||||
const bool is_flat_kv = (input->ne[1] == 1 && input->ne[2] == 1 && input->ne[3] == 1);
|
||||
if (!m_is_static && !is_flat_kv) {
|
||||
// do not fix ctx size to make llama-bench work across test params
|
||||
input_shape[2] = -1;
|
||||
}
|
||||
if (is_stateful()) {
|
||||
if (is_stateful() && !is_flat_kv) {
|
||||
// Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size]
|
||||
// to stateful layout [1, seq, n_heads_kv, head_size].
|
||||
assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 &&
|
||||
@@ -738,7 +827,9 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
|
||||
input_shape = ov::PartialShape{1, 1, 1, len};
|
||||
|
||||
} else if (is_inp_s_copy(input, op) || is_s_copy_leaf(input)) {
|
||||
input_shape = ov::PartialShape{1, 1, 1, -1};
|
||||
// On NPU the total slot count (n_seq_max) is fixed at translation time, so the s_copy
|
||||
// index list has a static length; on CPU/GPU it may change across compiles (defrag).
|
||||
input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1};
|
||||
|
||||
} else {
|
||||
input_shape = ov::PartialShape{get_shape(input)};
|
||||
@@ -790,13 +881,16 @@ void GgmlOvDecoder::add_extra_inputs() {
|
||||
// see llama_kv_cache_unified::get_n_kv and llama_kv_cache_unified::get_padding.
|
||||
// 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch
|
||||
|
||||
auto create_1d_input = [this](const std::string & name, int64_t value) {
|
||||
m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, !m_is_static};
|
||||
auto create_1d_input = [this](const std::string & name, int64_t value, bool force_parameter = false) {
|
||||
m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, force_parameter || !m_is_static};
|
||||
};
|
||||
|
||||
if (m_compute_params.attention_size != -1) {
|
||||
create_1d_input("attention_size", m_compute_params.attention_size);
|
||||
}
|
||||
if (m_compute_params.attention_size_static != -1) {
|
||||
create_1d_input("attention_size_static", m_compute_params.attention_size_static);
|
||||
}
|
||||
if (m_compute_params.attention_size_swa != -1) {
|
||||
create_1d_input("attention_size_swa", m_compute_params.attention_size_swa);
|
||||
}
|
||||
@@ -809,17 +903,32 @@ void GgmlOvDecoder::add_extra_inputs() {
|
||||
// create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active);
|
||||
|
||||
if (m_compute_params.cache_rs_reset_idx != -1) {
|
||||
create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx);
|
||||
create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len);
|
||||
// Whether/which cache slot to reset varies per compute call (e.g. a new sequence starting
|
||||
// vs. continued decoding). can_reuse_statically() does not invalidate the cached static
|
||||
// model on ComputeParams changes, so these must stay runtime Parameters even when static
|
||||
// (scale.cpp op_case 1 only uses them in value comparisons, never as Slice bounds, so this
|
||||
// does not reintroduce dynamic shapes).
|
||||
create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx, /*force_parameter=*/true);
|
||||
create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len, /*force_parameter=*/true);
|
||||
}
|
||||
|
||||
if (m_compute_params.s_copy_active_slot_len != -1) {
|
||||
create_1d_input("s_copy_active_slot_len", m_compute_params.s_copy_active_slot_len);
|
||||
if (m_is_static) {
|
||||
// Number of real tokens in the current prefill chunk. The last chunk is padded with
|
||||
// fabricated token ids; attention masks them out, but the recurrent (GDN/conv) path
|
||||
// would otherwise fold them into cache_r/cache_s permanently. Varies per chunk, so it
|
||||
// must stay a runtime Parameter; it is only compared against a Range or used as Gather
|
||||
// indices, so it does not make any shape dynamic.
|
||||
create_1d_input("chunk_valid_len", get_static_n_tokens(), /*force_parameter=*/true);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto & [node_name, writeback] : m_compute_params.rs_writebacks) {
|
||||
create_1d_input("rs_slot_begin_" + node_name, writeback.slot_begin);
|
||||
create_1d_input("rs_src_begin_" + node_name, writeback.src_begin);
|
||||
if (!m_is_static) {
|
||||
create_1d_input("rs_src_begin_" + node_name, writeback.src_begin);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1785,13 +1894,23 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
|
||||
auto dynamic_dim_stride = src_logical_nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) *
|
||||
ggml_type_size(node->type);
|
||||
int matched_dim_count = 0;
|
||||
int first_matched_dim = -1;
|
||||
for (int i = 0; i < GGML_MAX_DIMS; i++) {
|
||||
if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) {
|
||||
if (first_matched_dim == -1) {
|
||||
first_matched_dim = i;
|
||||
}
|
||||
m_node_dynamic_dims[node] = i;
|
||||
matched_dim_count++;
|
||||
}
|
||||
}
|
||||
if (matched_dim_count != 1) {
|
||||
if (matched_dim_count > 1 && node->src[0]->ne[dynamic_dim_idx] == 1) {
|
||||
// Single-token capture: every trailing dim is size 1 with the same stride, so
|
||||
// the match is ambiguous. The lowest index is the real axis; the rest are
|
||||
// ggml's size-1 padding. Bailing out here would bake the captured token count
|
||||
// into the static prefill model, which then runs with a different one.
|
||||
m_node_dynamic_dims[node] = first_matched_dim;
|
||||
} else if (matched_dim_count != 1) {
|
||||
m_node_dynamic_dims[node] = -1;
|
||||
GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for CONT node '%s', src[0]: '%s'\n",
|
||||
node->name, node->src[0]->name);
|
||||
|
||||
@@ -47,6 +47,7 @@ struct ComputeParams {
|
||||
int seq_active_start = 0;
|
||||
int attention_size = -1;
|
||||
int attention_size_swa = -1;
|
||||
int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper)
|
||||
int input_len = -1;
|
||||
int token_len_per_seq = -1;
|
||||
int past_kv_len = -1;
|
||||
@@ -84,14 +85,15 @@ struct ComputeParams {
|
||||
|
||||
struct RsWriteback {
|
||||
int slot_begin = 0; // first cache slot written by the CPY
|
||||
int src_begin = 0; // where the copied data starts in the source tensor (in rows of it)
|
||||
int src_begin = 0; // first source row or column copied by the CPY
|
||||
};
|
||||
|
||||
std::map<std::string, RsWriteback> rs_writebacks;
|
||||
// Offsets of the state cache writeback CPY nodes, keyed by node name. They change with the
|
||||
// batch (kv head, active sequence count, token count) and, with rollback enabled
|
||||
// (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot, each snapshot
|
||||
// taking a different conv_input window. Passed to the cached model as runtime inputs.
|
||||
// Destination slot offset of each state cache writeback CPY node, keyed by node name. It
|
||||
// changes with the batch (kv head, active sequence count) and, with rollback enabled
|
||||
// (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot. Passed to the
|
||||
// cached model as a runtime input. Dynamic models also receive the source-side offset; static
|
||||
// models use a fixed end-anchored offset in the translator.
|
||||
};
|
||||
|
||||
class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
|
||||
|
||||
@@ -32,6 +32,8 @@ void ggml_openvino_device_config::init() {
|
||||
"GGML_OPENVINO_DEVICE",
|
||||
"GGML_OPENVINO_CACHE_DIR",
|
||||
"GGML_OPENVINO_DEBUG_NODE",
|
||||
"GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR",
|
||||
"GGML_OPENVINO_NPU_COMPILE_CONFIG",
|
||||
// Integer values (use ggml_openvino_getenv_int)
|
||||
"GGML_OPENVINO_PREFILL_CHUNK_SIZE",
|
||||
// Boolean toggles (treated as int flags via ggml_openvino_getenv_int)
|
||||
@@ -41,6 +43,9 @@ void ggml_openvino_device_config::init() {
|
||||
"GGML_OPENVINO_DUMP_IR",
|
||||
"GGML_OPENVINO_DEBUG_INPUT",
|
||||
"GGML_OPENVINO_DEBUG_OUTPUT",
|
||||
// Force the static (NPU-shape) compute path on any device, e.g. GGML_OPENVINO_DEVICE=CPU,
|
||||
// to test the static-shape translation without NPUW/real NPU hardware in the loop.
|
||||
"GGML_OPENVINO_FORCE_STATIC",
|
||||
"GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS",
|
||||
"GGML_OPENVINO_ENABLE_CACHE",
|
||||
"GGML_OPENVINO_DISABLE_CACHE",
|
||||
@@ -50,7 +55,7 @@ void ggml_openvino_device_config::init() {
|
||||
"GGML_OPENVINO_MEMORY_OPTIMIZE",
|
||||
"GGML_OPENVINO_RELEASE_WEIGHTS",
|
||||
"GGML_OPENVINO_REDUCE_COMPILE_MEM",
|
||||
"GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR",
|
||||
"GGML_OPENVINO_LOG_UNSUPPORTED_OPS",
|
||||
};
|
||||
|
||||
for (const char * const & env_var : env_var_names) {
|
||||
@@ -85,6 +90,11 @@ void ggml_openvino_device_config::init() {
|
||||
compile_config["NPUW_CACHE_DIR"] = cache_dir;
|
||||
compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
|
||||
}
|
||||
const char * compilation_mode_params =
|
||||
ggml_openvino_getenv_str("GGML_OPENVINO_NPU_COMPILE_CONFIG");
|
||||
if (compilation_mode_params && strlen(compilation_mode_params) > 0) {
|
||||
compile_config["NPU_COMPILATION_MODE_PARAMS"] = compilation_mode_params;
|
||||
}
|
||||
} else if (cache_dir && strlen(cache_dir) > 0) {
|
||||
compile_config.insert(ov::cache_dir(cache_dir));
|
||||
compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
|
||||
|
||||
@@ -908,11 +908,27 @@ static bool has_non_contiguous_view_input(const ggml_tensor * op) {
|
||||
}
|
||||
|
||||
static bool is_supported_flash_attn_pattern(const ggml_tensor * op) {
|
||||
// pattern of q,k,v should be q->op==PERMUTE, q->src[0]->op==VIEW, q->src[0]->src[0]->view_src==nullptr
|
||||
// Each Q/K/V input must follow one of:
|
||||
// PERMUTE -> VIEW -> base (view_src==nullptr) (llama KV-cache path)
|
||||
// PERMUTE -> RESHAPE -> base (view_src==nullptr) (whisper Q)
|
||||
// VIEW -> base (view_src==nullptr) (whisper K/V from kv_pad)
|
||||
for (int i = 0; i < 3; i++) {
|
||||
const ggml_tensor * src = op->src[i];
|
||||
if (src->op != GGML_OP_PERMUTE || src->src[0] == nullptr || src->src[0]->op != GGML_OP_VIEW ||
|
||||
src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) {
|
||||
if (src->op == GGML_OP_PERMUTE) {
|
||||
if (src->src[0] == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (src->src[0]->op != GGML_OP_VIEW && src->src[0]->op != GGML_OP_RESHAPE) {
|
||||
return false;
|
||||
}
|
||||
if (src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) {
|
||||
return false;
|
||||
}
|
||||
} else if (src->op == GGML_OP_VIEW) {
|
||||
if (src->src[0] == nullptr || src->src[0]->view_src != nullptr) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1030,18 +1046,29 @@ static bool is_msa_block_mask_expansion(const ggml_tensor * op) {
|
||||
return tensor_name_starts_with(src, "msa_block_mask");
|
||||
}
|
||||
|
||||
static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
namespace {
|
||||
struct ggml_openvino_op_support {
|
||||
bool is_supported = true;
|
||||
std::string reason;
|
||||
|
||||
operator bool() const {
|
||||
return is_supported;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
if (is_msa_block_mask_expansion(op)) {
|
||||
return true;
|
||||
return {false, "MSA block mask expansion is not supported"};
|
||||
}
|
||||
|
||||
switch (op->op) {
|
||||
case GGML_OP_CONCAT: {
|
||||
if (op->type == GGML_TYPE_I64) {
|
||||
return true;
|
||||
return {false, "CONCAT with I64 type is not supported"};
|
||||
}
|
||||
if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_BF16 && has_view_op_input(op)) {
|
||||
return true;
|
||||
return {false, "CONCAT with BF16 type and VIEW input is not supported on GPU"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1052,24 +1079,21 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
|
||||
// OpenVINO SET translation currently supports dst layouts that match src0 strides.
|
||||
if (op->src[0] == nullptr || nb1 != op->src[0]->nb[1] || nb2 != op->src[0]->nb[2] || nb3 != op->src[0]->nb[3]) {
|
||||
// std::cout << "Unsupported SET op with dst nb1=" << nb1 << ", nb2=" << nb2 << ", nb3=" << nb3
|
||||
// << " that does not match src0 strides nb[1]="
|
||||
// << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null")
|
||||
// << ", nb[2]=" << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null")
|
||||
// << ", nb[3]=" << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")
|
||||
// << std::endl;
|
||||
return true;
|
||||
return {false, "SET op with dst nb1=" + std::to_string(nb1) + ", nb2=" + std::to_string(nb2) + ", nb3=" + std::to_string(nb3) +
|
||||
" that does not match src0 strides nb[1]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null") +
|
||||
", nb[2]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null") +
|
||||
", nb[3]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_GET_ROWS:
|
||||
case GGML_OP_SET_ROWS: {
|
||||
if (op->ne[3] != 1) {
|
||||
return true;
|
||||
return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
|
||||
}
|
||||
if (op->op == GGML_OP_GET_ROWS && ggml_openvino_get_device_name() == "GPU" &&
|
||||
op->src[0]->type == GGML_TYPE_BF16) {
|
||||
return true;
|
||||
return {false, "GET_ROWS with BF16 src0 is not supported on GPU"};
|
||||
}
|
||||
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
|
||||
op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) {
|
||||
@@ -1078,14 +1102,14 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
// make_int8_weights/make_int4_weights: dequant is done in f16, not f32, to keep the
|
||||
// Convert/Subtract/Multiply chain fusable into GatherMatmulCompressed/FullyConnectedCompressed
|
||||
// for the shared non-test code paths).
|
||||
return true;
|
||||
return {false, "GET_ROWS/SET_ROWS with ne[0] == 256 and type " + std::string(ggml_type_name(op->src[0]->type)) +
|
||||
" rejected due to f16-arithmetic dequant rounding errors that intermittently exceed 1e-7 NMSE threshold"};
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case GGML_OP_RESHAPE: {
|
||||
if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) {
|
||||
return true;
|
||||
return {false, "RESHAPE for ffn_norm_exps is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1093,11 +1117,13 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
case GGML_OP_MUL:
|
||||
case GGML_OP_SUB: {
|
||||
if (op->src[1]->op == GGML_OP_PERMUTE) {
|
||||
return true;
|
||||
return {false, "ADD/MUL/SUB with PERMUTE src1 is not supported"};
|
||||
}
|
||||
for (int i = 0; i < 4; i++) {
|
||||
if (op->src[0]->ne[i] != op->src[1]->ne[i] && (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1)) {
|
||||
return true;
|
||||
return {false, "ADD/MUL/SUB with incompatible broadcast shapes: src0->ne[" + std::to_string(i) + "]=" +
|
||||
std::to_string(op->src[0]->ne[i]) + ", src1->ne[" + std::to_string(i) + "]=" +
|
||||
std::to_string(op->src[1]->ne[i])};
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -1106,7 +1132,7 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
// Keep support aligned with the CPU backend implementation, which only handles f32 inputs/output and i32 ids.
|
||||
if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32 ||
|
||||
op->src[2]->type != GGML_TYPE_I32) {
|
||||
return true;
|
||||
return {false, "ADD_ID only supports F32 inputs/output and I32 ids"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1116,14 +1142,27 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
// until the fused GPU kernel is reliable. (falied case llama-arch-test mpt)
|
||||
if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->ne[0] == op->ne[0] &&
|
||||
op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1) {
|
||||
return true;
|
||||
return {false, "DIV per-channel scale broadcast is not supported on GPU"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_POOL_2D: {
|
||||
const auto& name = ggml_openvino_get_device_name();
|
||||
if (name == "GPU") {
|
||||
const int32_t * params = op->op_params;
|
||||
const int k0 = params[1];
|
||||
const int k1 = params[2];
|
||||
const int p0 = params[5];
|
||||
const int p1 = params[6];
|
||||
if ((p0 > 0 || p1 > 0) && (k0 < 3 || k1 < 3)) {
|
||||
return {false, "POOL_2D with padding and kernel size < 3 is not supported on " + name};
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_SUM_ROWS: {
|
||||
// if the input is PERMUTE skip
|
||||
if (op->src[0]->op == GGML_OP_PERMUTE) {
|
||||
return true;
|
||||
return {false, "SUM_ROWS with PERMUTE input is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1140,54 +1179,51 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
// accuracy drift in the OpenVINO path. Restrict by scale=1.0 to avoid
|
||||
// affecting non-gemma3n models such as Llama-3.2.
|
||||
if (fabsf(scale - 1.0f) < 1e-6f && is_gemma3n_flash_attn_pattern(op)) {
|
||||
return true;
|
||||
return {false, "FLASH_ATTN_EXT gemma3n pattern on GPU is not supported"};
|
||||
}
|
||||
|
||||
if (op->src[4] != nullptr) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n");
|
||||
return true;
|
||||
return {false, "FLASH_ATTN_EXT with sinks is not supported"};
|
||||
}
|
||||
if (!is_supported_flash_attn_pattern(op)) {
|
||||
return true;
|
||||
return {false, "FLASH_ATTN_EXT unsupported attention pattern"};
|
||||
}
|
||||
if (max_bias > 0) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with max_bias > 0\n");
|
||||
return true;
|
||||
return {false, "FLASH_ATTN_EXT with max_bias > 0 (max_bias=" + std::to_string(max_bias) + ") is not supported"};
|
||||
}
|
||||
if (logit_softcap != 0) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with logit_softcap != 0\n");
|
||||
return true;
|
||||
return {false, "FLASH_ATTN_EXT with logit_softcap != 0 (logit_softcap=" + std::to_string(logit_softcap) + ") is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_PERMUTE: {
|
||||
if (op->type == GGML_TYPE_BF16) {
|
||||
// err msg: [GPU] Could not find a suitable kernel for transpose
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support PERMUTE with BF16 type\n");
|
||||
return true;
|
||||
if (op->type == GGML_TYPE_BF16 && ggml_openvino_get_device_name() == "GPU") {
|
||||
return {false, "PERMUTE with BF16 type is not supported on GPU"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_CPY: {
|
||||
if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support CPY with non-contiguous data or bf16 types\n");
|
||||
return true;
|
||||
return {false, "CPY with BF16 src type is not supported"};
|
||||
}
|
||||
// CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend.
|
||||
if (ggml_is_quantized(op->type)) {
|
||||
return true;
|
||||
return {false, "CPY to quantized destination (e.g. f32 -> q4_0) is numerically unstable"};
|
||||
}
|
||||
if (ggml_nelements(op->src[0]) != ggml_nelements(op->src[1])) {
|
||||
return true;
|
||||
return {false, "CPY with mismatched element counts is not supported: src0=" + std::to_string(ggml_nelements(op->src[0])) +
|
||||
" != src1=" + std::to_string(ggml_nelements(op->src[1]))};
|
||||
}
|
||||
// op test case with non-contiguous src or dst
|
||||
if ((op->ne[0] == 3 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
|
||||
(op->ne[0] == 1 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
|
||||
(op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) {
|
||||
return true;
|
||||
return {false, "CPY with non-contiguous shape [" + std::to_string(op->ne[0]) + ", " +
|
||||
std::to_string(op->ne[1]) + ", " + std::to_string(op->ne[2]) + ", " +
|
||||
std::to_string(op->ne[3]) + "] is not supported"};
|
||||
}
|
||||
if (!cpy_output_view_is_supported(op)) {
|
||||
return true;
|
||||
return {false, "CPY with non-contiguous output view is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1196,13 +1232,14 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
ggml_is_quantized(op->src[0]->type) && strcmp(op->src[0]->name, "a") == 0 &&
|
||||
strcmp(op->src[1]->name, "b") == 0 && op->src[0]->ne[1] == 1 && op->src[1]->ne[1] == 64 &&
|
||||
op->src[0]->ne[0] == 256 && op->src[1]->ne[0] == 256) {
|
||||
return true;
|
||||
return {false, "MUL_MAT quantized benchmark test case on GPU is not supported"};
|
||||
}
|
||||
if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) {
|
||||
return true;
|
||||
return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) +
|
||||
", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])};
|
||||
}
|
||||
if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) {
|
||||
return true;
|
||||
return {false, "MUL_MAT with both inputs as VIEW is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1210,16 +1247,17 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
// Single-expert (or empty) MUL_MAT_ID is a degenerate shape that stresses GatherMatmul edge
|
||||
// cases and never occurs in real MoE; let it fall back to CPU.
|
||||
if (op->src[0] != nullptr && op->src[0]->ne[2] <= 1) {
|
||||
return true;
|
||||
return {false, "MUL_MAT_ID with single-expert or empty ne[2] <= 1 (ne[2]=" +
|
||||
std::to_string(op->src[0]->ne[2]) + ") is not supported"};
|
||||
}
|
||||
if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_BF16) {
|
||||
return true;
|
||||
return {false, "MUL_MAT_ID with BF16 weights on GPU is not supported"};
|
||||
}
|
||||
// GPU MUL_MAT_ID uses a Gather+MatMul fallback because the GPU plugin rejects internal
|
||||
// GatherMatmul for these test shapes. Skip cases that would materialize a large selected
|
||||
// expert-weight temporary.
|
||||
if (ggml_openvino_get_device_name() == "GPU" && mul_mat_id_requires_large_tmp(op)) {
|
||||
return true;
|
||||
return {false, "MUL_MAT_ID requires large temporary on GPU"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1229,51 +1267,46 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
const int mode = op_params[2];
|
||||
if (op_params[15] != 0) {
|
||||
// FIXME: support ggml_rope_set_offset
|
||||
return true;
|
||||
return {false, "ggml_rope_set_offset is not supported"};
|
||||
}
|
||||
if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_IMROPE) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support ROPE with mode %d\n", mode);
|
||||
return true;
|
||||
return {false, "ROPE with mode " + std::to_string(mode) + " is not supported"};
|
||||
}
|
||||
const int64_t head_dim = op->src[0]->ne[0];
|
||||
const int64_t rope_dims = n_dims == 0 ? head_dim : n_dims;
|
||||
if (rope_dims <= 0 || rope_dims > head_dim || (rope_dims % 2) != 0) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support ROPE with n_dims %d and src[0]->ne[0] %ld\n", n_dims,
|
||||
// op->src[0]->ne[0]);
|
||||
return true;
|
||||
return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", head_dim=" + std::to_string(head_dim) + " is not supported"};
|
||||
}
|
||||
if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support ROPE with type %s\n", ggml_type_name(op->type));
|
||||
return true;
|
||||
return {false, "ROPE with type " + std::string(ggml_type_name(op->type)) + " is not supported"};
|
||||
}
|
||||
if (op->src[0]->op == GGML_OP_VIEW) {
|
||||
if (op->src[0]->view_src->ne[1] != op->src[0]->ne[2]) {
|
||||
// GGML_LOG_WARN(
|
||||
// "OpenVINO backend does not support ROPE with src[0]->view_src->ne[1] %ld != src[0]->ne[2] "
|
||||
// "%ld\n",
|
||||
// op->src[0]->view_src->ne[1], op->src[0]->ne[2]);
|
||||
return true;
|
||||
const struct ggml_tensor * view = op->src[0];
|
||||
const struct ggml_tensor * view_src = view->view_src;
|
||||
if (view_src->ne[1] != view->ne[1] || view_src->ne[2] != view->ne[2] || view_src->ne[3] != view->ne[3]) {
|
||||
return {false, "ROPE with view_src->ne [" + std::to_string(view_src->ne[1]) + ", " +
|
||||
std::to_string(view_src->ne[2]) + ", " + std::to_string(view_src->ne[3]) +
|
||||
"] != view->ne [" + std::to_string(view->ne[1]) + ", " +
|
||||
std::to_string(view->ne[2]) + ", " + std::to_string(view->ne[3]) +
|
||||
"] is not supported"};
|
||||
}
|
||||
}
|
||||
if (mode == GGML_ROPE_TYPE_IMROPE &&
|
||||
(op->src[2] != 0 || ((const float *) op_params)[6] != 1 || ((const float *) op_params)[7] != 0 ||
|
||||
((const float *) op_params)[8] != 1)) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor\n");
|
||||
return true;
|
||||
return {false, "IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_TRANSPOSE: {
|
||||
// if the type is bf16, will return true
|
||||
if (op->type == GGML_TYPE_BF16) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support CONT with BF16 type\n");
|
||||
return true;
|
||||
return {false, "TRANSPOSE with BF16 type is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_REPEAT: {
|
||||
if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_BF16) {
|
||||
return true;
|
||||
return {false, "REPEAT with BF16 type is not supported on GPU"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1285,15 +1318,15 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
// return true;
|
||||
// }
|
||||
if (op->src[2]->op == GGML_OP_PERMUTE) {
|
||||
return true;
|
||||
return {false, "GATED_DELTA_NET with PERMUTE src2 is not supported"};
|
||||
}
|
||||
// kda (per-key-dimension gating) not supported by fused GatedDeltaNet op
|
||||
if (op->src[3]->ne[0] != 1) {
|
||||
return true;
|
||||
return {false, "GATED_DELTA_NET with kda (per-key-dimension gating) is not supported"};
|
||||
}
|
||||
// K > 1 (multiple state snapshots) not supported by fused op
|
||||
if (((const int32_t *) op->op_params)[0] > 1) {
|
||||
return true;
|
||||
return {false, "GATED_DELTA_NET with K > 1 (multiple state snapshots) is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1307,17 +1340,17 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
|
||||
// Skip TOPK_MOE fused tests until it is fully supported.
|
||||
// The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe.
|
||||
if (strcmp(op->name, "selected_experts") == 0) {
|
||||
return true;
|
||||
return {false, "VIEW for selected_experts (argsort_top_k) is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
return {true, ""};
|
||||
}
|
||||
|
||||
static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
|
||||
static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(ggml_backend_dev_t dev, const ggml_tensor * op) {
|
||||
GGML_ASSERT(dev->reg != nullptr);
|
||||
|
||||
static std::unordered_set<ggml_type> supported_types{
|
||||
@@ -1367,48 +1400,41 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
|
||||
case GGML_OP_UNARY: {
|
||||
auto supported = supported_unary_ops.find(ggml_get_unary_op(op)) != supported_unary_ops.end();
|
||||
if (!supported) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support unary op %s\n", ggml_unary_op_name(ggml_get_unary_op(op)));
|
||||
return false;
|
||||
return {false, "unary op " + std::string(ggml_unary_op_name(ggml_get_unary_op(op))) + " has no op translator"};
|
||||
}
|
||||
if (ggml_get_unary_op(op) == GGML_UNARY_OP_EXP && op->type == GGML_TYPE_F32) {
|
||||
return false;
|
||||
return {false, "UNARY_EXP with F32 type is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_GLU: {
|
||||
auto supported = supported_glu_ops.find(ggml_get_glu_op(op)) != supported_glu_ops.end();
|
||||
if (!supported) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op)));
|
||||
return false;
|
||||
return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " has no op translator"};
|
||||
}
|
||||
// if (has_view_op_input(op)) {
|
||||
// // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
|
||||
// // ggml_glu_op_name(ggml_get_glu_op(op)));
|
||||
// return false;
|
||||
// return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " with view input is not supported"};
|
||||
// }
|
||||
if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) {
|
||||
// triggers bug in ov gpu
|
||||
return false;
|
||||
return {false, "GLU op with odd src0 ne[0] and null src1 is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
auto supported = supported_ops.find(op->op) != supported_ops.end();
|
||||
if (!supported) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support op %s\n", ggml_op_name(op->op));
|
||||
return false;
|
||||
return {false, "op " + std::string(ggml_op_name(op->op)) + " has no op translator"};
|
||||
}
|
||||
static std::set<ggml_op> ops_not_support_view_input{};
|
||||
if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support op %s with view input\n", ggml_op_name(op->op));
|
||||
return false;
|
||||
return {false, "op " + std::string(ggml_op_name(op->op)) + " with VIEW input is not supported"};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (supported_types.find(op->type) == supported_types.end()) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(op->type));
|
||||
return false;
|
||||
return {false, "tensor type " + std::string(ggml_type_name(op->type)) + " is not supported"};
|
||||
}
|
||||
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
||||
auto * src = op->src[i];
|
||||
@@ -1416,21 +1442,32 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
|
||||
break;
|
||||
}
|
||||
if (supported_types.find(src->type) == supported_types.end()) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type));
|
||||
return false;
|
||||
return {false, "src[" + std::to_string(i) + "] type " + std::string(ggml_type_name(src->type)) + " is not supported"};
|
||||
}
|
||||
const bool is_supported_3d_moe_expert =
|
||||
op->op == GGML_OP_MUL_MAT_ID && i == 0 && (src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1);
|
||||
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) {
|
||||
// GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n");
|
||||
return false;
|
||||
return {false, "3D quantized tensor for src[" + std::to_string(i) + "] is not supported"};
|
||||
}
|
||||
}
|
||||
|
||||
if (is_op_unsupported_case(op)) {
|
||||
return false;
|
||||
auto op_support_case = is_op_supported_case(op);
|
||||
if (!op_support_case.is_supported) {
|
||||
return op_support_case;
|
||||
}
|
||||
return true;
|
||||
return {true, ""};
|
||||
}
|
||||
|
||||
static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
|
||||
auto res = ggml_backend_openvino_device_supports_op_impl(dev, op);
|
||||
if (!res.is_supported) {
|
||||
static const bool log_unsupported = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_UNSUPPORTED_OPS") != 0;
|
||||
if (log_unsupported) {
|
||||
GGML_LOG_WARN("OpenVINO op unsupported: op '%s' (%s), type %s: %s\n",
|
||||
op->name, ggml_op_name(op->op), ggml_type_name(op->type), res.reason.c_str());
|
||||
}
|
||||
}
|
||||
return res.is_supported;
|
||||
}
|
||||
|
||||
static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
|
||||
|
||||
@@ -3,8 +3,11 @@
|
||||
#include "../utils.h"
|
||||
|
||||
#include <climits>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <numeric>
|
||||
#include <openvino/frontend/exception.hpp>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
@@ -12,9 +15,14 @@
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/negative.hpp>
|
||||
#include <openvino/op/range.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/scatter_update.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/subtract.hpp>
|
||||
#include <vector>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
@@ -61,10 +69,27 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
// Recurrent state cache writeback into a slot block of the cache. Where the block starts and
|
||||
// where the copied data starts in the source are runtime inputs, so the cached model works for
|
||||
// any kv head, active sequence count and token count. The result is the full updated cache.
|
||||
// Recurrent state cache writeback into a slot block of the cache. Where the block starts is a
|
||||
// runtime input, so the cached model works for any kv head and active sequence count. The
|
||||
// result is the full updated cache.
|
||||
// op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder.
|
||||
if (op_case == 3) {
|
||||
// With -np 1 (and generally whenever there is no defrag remainder) this GET_ROWS gathers
|
||||
// zero rows: nothing to write back, and the cache is unchanged. NPU rejects zero-size
|
||||
// tensors, so short-circuit instead of building a degenerate Slice/Concat chain.
|
||||
bool is_empty = false;
|
||||
if (input_shape.rank().is_static()) {
|
||||
for (const auto & d : input_shape) {
|
||||
if (d.is_static() && d.get_length() == 0) {
|
||||
is_empty = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (is_empty) {
|
||||
return {context.get_input(1)};
|
||||
}
|
||||
}
|
||||
const std::string slot_begin_name = "rs_slot_begin_" + context.get_name();
|
||||
const bool slice_assign =
|
||||
context.has_input(slot_begin_name) && !context.is_stateful() && (op_case >= 1 && op_case <= 3);
|
||||
@@ -81,19 +106,49 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
ov::Output<ov::Node> begin = context.get_input(slot_begin_name);
|
||||
auto base = context.get_input(1);
|
||||
if (op_case == 1) {
|
||||
// GDN packs [attn | state snapshots]; the state part runs from src_begin to the end.
|
||||
auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
|
||||
auto state_part = std::make_shared<ov::op::v8::Slice>(context.get_input(0), src_begin, int_max, one, axis);
|
||||
ov::Output<ov::Node> state_begin;
|
||||
const std::string src_begin_name = "rs_src_begin_" + context.get_name();
|
||||
if (context.has_input(src_begin_name)) {
|
||||
state_begin = context.get_input(src_begin_name);
|
||||
} else {
|
||||
auto ssm_state_size = context.get_ssm_state_size();
|
||||
if (context.has_input("s_copy_active_slot_len")) {
|
||||
auto len = context.get_input("s_copy_active_slot_len");
|
||||
auto state_rows = std::make_shared<ov::op::v1::Multiply>(
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {ssm_state_size}), len);
|
||||
state_begin = std::make_shared<ov::op::v0::Negative>(state_rows);
|
||||
} else {
|
||||
state_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-ssm_state_size});
|
||||
}
|
||||
}
|
||||
auto state_part =
|
||||
std::make_shared<ov::op::v8::Slice>(context.get_input(0), state_begin, int_max, one, axis);
|
||||
src = std::make_shared<ov::op::v1::Reshape>(state_part, feature, false);
|
||||
} else if (op_case == 2) {
|
||||
// conv_input is [previous conv state | new tokens]; copy the conv_kernel_size - 1 wide
|
||||
// window starting at src_begin, which is the snapshot this writeback corresponds to.
|
||||
// conv_input is [previous conv state | new tokens]; the snapshot is the conv_kernel_size - 1
|
||||
// columns ending at the last *valid* token. Gather (rather than Slice) keeps the output
|
||||
// shape static even though the window start is a runtime value.
|
||||
auto window_size = (int64_t) input_shape[3].get_length();
|
||||
auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
|
||||
auto src_end = std::make_shared<ov::op::v1::Add>(
|
||||
src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
|
||||
auto window = std::make_shared<ov::op::v8::Slice>(context.get_input(0), src_begin, src_end, one,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
|
||||
ov::Output<ov::Node> window;
|
||||
auto col_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
|
||||
const std::string src_begin_name = "rs_src_begin_" + context.get_name();
|
||||
if (context.has_input(src_begin_name)) {
|
||||
auto src_begin = context.get_input(src_begin_name);
|
||||
auto src_end = std::make_shared<ov::op::v1::Add>(
|
||||
src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
|
||||
window = std::make_shared<ov::op::v8::Slice>(context.get_input(0), src_begin, src_end, one, col_axis);
|
||||
} else if (context.has_input("chunk_valid_len")) {
|
||||
std::vector<int64_t> offsets(window_size);
|
||||
std::iota(offsets.begin(), offsets.end(), 0);
|
||||
auto indices = std::make_shared<ov::op::v1::Add>(
|
||||
ov::op::v0::Constant::create(ov::element::i64, {(size_t) window_size}, offsets),
|
||||
context.get_input("chunk_valid_len"));
|
||||
window = std::make_shared<ov::op::v8::Gather>(context.get_input(0), indices, col_axis);
|
||||
} else {
|
||||
auto window_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-window_size});
|
||||
window =
|
||||
std::make_shared<ov::op::v8::Slice>(context.get_input(0), window_begin, int_max, one, col_axis);
|
||||
}
|
||||
const auto base_shape = base.get_partial_shape();
|
||||
FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape.rank().get_length() == 4,
|
||||
"CPY conv state cache update requires rank-4 base cache");
|
||||
@@ -157,6 +212,63 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
|
||||
auto input = process_view_input_new(context, 0);
|
||||
|
||||
if (op_case == 5 || op_case == 6) {
|
||||
auto input_shape = context.get_input_shape(0);
|
||||
auto output_shape = context.get_output_shape();
|
||||
auto dst_ggml_shape = context.get_view_input_ggml_shape(1, 0);
|
||||
auto dst_stride = context.get_view_input_stride(1, 0);
|
||||
size_t offset_bytes = context.get_view_input_offset(1, 0);
|
||||
auto n_state = (int64_t) context.get_input_shape(0)[3].get_length();
|
||||
auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state});
|
||||
auto kv_buf = context.get_input(1); // shape {1,1,1,N}
|
||||
|
||||
Output<Node> token_len_per_seq;
|
||||
Output<Node> n_write_dyn;
|
||||
if (context.has_input("token_len_per_seq")) {
|
||||
token_len_per_seq = context.get_input("token_len_per_seq");
|
||||
n_write_dyn = std::make_shared<ov::op::v1::Multiply>(token_len_per_seq, n_state_c);
|
||||
} else {
|
||||
n_write_dyn = ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) dst_ggml_shape[3]});
|
||||
}
|
||||
size_t elem_size = dst_stride[3];
|
||||
FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY KV cache view update has invalid element size");
|
||||
int64_t start_elem = (int64_t) (offset_bytes / elem_size);
|
||||
// op_case 5: decoder self-attention – write offset advances each step.
|
||||
// op_case 6: encoder self-attn or cross-attn – offset fixed at compile time.
|
||||
const bool is_decoder_self_attn = (op_case == 5);
|
||||
auto ones_c = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{1, 1, 1});
|
||||
auto new_shape = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{ones_c, n_write_dyn}, 0);
|
||||
|
||||
auto reshaped = std::make_shared<ov::op::v1::Reshape>(input, new_shape, false);
|
||||
auto data = std::make_shared<ov::op::v0::Convert>(reshaped, context.get_output_type());
|
||||
// Indices [start_elem .. start_elem + n_write) on axis 3 of {1,1,1,N}
|
||||
// For decoder self-attention the write offset advances each step, so compute it
|
||||
// dynamically from the model inputs: start = (attention_size - token_len_per_seq) * n_state.
|
||||
// For encoder self-attn and cross-attn the offset is fixed at graph-compile time.
|
||||
ov::Output<ov::Node> start;
|
||||
if (is_decoder_self_attn && context.has_input("attention_size") && context.has_input("token_len_per_seq")) {
|
||||
auto attention_size_in = context.get_input("attention_size");
|
||||
auto token_len_in = context.get_input("token_len_per_seq");
|
||||
auto past_tokens = std::make_shared<ov::op::v1::Subtract>(attention_size_in, token_len_in);
|
||||
auto new_start = std::make_shared<ov::op::v1::Multiply>(past_tokens, n_state_c);
|
||||
start = std::make_shared<ov::op::v1::Add>(
|
||||
new_start, ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem}));
|
||||
} else {
|
||||
start = ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem});
|
||||
}
|
||||
auto start_squeezed = std::make_shared<ov::op::v0::Squeeze>(start);
|
||||
auto end = std::make_shared<ov::op::v1::Add>(start_squeezed, n_write_dyn);
|
||||
auto end_squeezed = std::make_shared<ov::op::v0::Squeeze>(end);
|
||||
auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
auto step_squeezed = std::make_shared<ov::op::v0::Squeeze>(step);
|
||||
auto indices =
|
||||
std::make_shared<ov::op::v4::Range>(start_squeezed, end_squeezed, step_squeezed, ov::element::i64);
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
|
||||
|
||||
auto kv_updated = std::make_shared<ov::op::v3::ScatterUpdate>(kv_buf, indices, data, axis);
|
||||
return rename_outputs_with_suffix({kv_updated}, context.get_name());
|
||||
}
|
||||
|
||||
if (input_shape != output_shape) {
|
||||
auto new_shape = ov::op::v0::Constant::create(
|
||||
ov::element::i64, {static_cast<size_t>(output_shape.rank().get_length())}, output_shape.to_shape());
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
#include "../utils.h"
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <memory>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/broadcast.hpp>
|
||||
@@ -15,6 +15,7 @@
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/scaled_dot_product_attention.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/softmax.hpp>
|
||||
#include <openvino/op/transpose.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
@@ -24,13 +25,62 @@ namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace op {
|
||||
static ov::Output<ov::Node> reshape_flat_kv(const ov::Output<ov::Node> & kv_flat,
|
||||
size_t view_offset_bytes,
|
||||
size_t nb1_bytes,
|
||||
int64_t n_head,
|
||||
int64_t head_size,
|
||||
const ov::Output<ov::Node> & attention_size) {
|
||||
int64_t n_state = n_head * head_size;
|
||||
int64_t layer_start_elem = (int64_t) (view_offset_bytes / (nb1_bytes / n_state));
|
||||
// Dynamic slice: [layer_start_elem, layer_start_elem + n_kv * n_state)
|
||||
auto start_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {layer_start_elem});
|
||||
auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state});
|
||||
// end = start + attention_size * n_state (both static + dynamic)
|
||||
auto kv_len_elems = std::make_shared<ov::op::v1::Multiply>(attention_size, n_state_c);
|
||||
auto end_c = std::make_shared<ov::op::v1::Add>(start_c, kv_len_elems);
|
||||
auto step_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
auto axis_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
|
||||
auto sliced = std::make_shared<ov::op::v8::Slice>(kv_flat, start_c, end_c, step_c, axis_c);
|
||||
|
||||
// KV cache is laid out as {n_kv, n_head, head_size} in memory
|
||||
// Reshape to {1, n_kv, n_head, head_size}, then transpose to {1, n_head, n_kv, head_size}
|
||||
// as required by SDPA.
|
||||
auto one_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
auto n_head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_head});
|
||||
auto head_size_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_size});
|
||||
// reshape: {n_kv*n_state} -> {1, n_kv, n_head, head_size}
|
||||
auto new_shape =
|
||||
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{one_c, attention_size, n_head_c, head_size_c}, 0);
|
||||
auto reshaped = std::make_shared<ov::op::v1::Reshape>(sliced, new_shape, false);
|
||||
// transpose: {1, n_kv, n_head, head_size} -> {1, n_head, n_kv, head_size}
|
||||
auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3});
|
||||
auto ret = std::make_shared<ov::op::v1::Transpose>(reshaped, perm);
|
||||
return ret;
|
||||
}
|
||||
|
||||
OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
num_inputs_check(context, 4, 4);
|
||||
num_inputs_check(context, 3, 4);
|
||||
const bool has_mask = context.get_input_size() == 4;
|
||||
auto q_f32 = context.get_input(0);
|
||||
auto k = context.get_input(1);
|
||||
auto v = context.get_input(2);
|
||||
auto mask = context.get_input(3);
|
||||
const int op_case = context.get_op_case();
|
||||
|
||||
if (op_case == 1 || op_case == 2) {
|
||||
int64_t n_state_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[3];
|
||||
int64_t n_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[1];
|
||||
size_t nb1 = context.get_view_input_stride(1, 0)[2];
|
||||
size_t offset = context.get_view_input_offset(1, 0);
|
||||
ov::Output<ov::Node> attention_size;
|
||||
if (op_case == 1) {
|
||||
attention_size = context.get_input("attention_size");
|
||||
} else {
|
||||
attention_size = context.get_input("attention_size_static");
|
||||
}
|
||||
k = reshape_flat_kv(k, offset, nb1, n_head, n_state_head, attention_size);
|
||||
v = reshape_flat_kv(v, offset, nb1, n_head, n_state_head, attention_size);
|
||||
}
|
||||
|
||||
float * params = reinterpret_cast<float *>(context.get_output_op_params());
|
||||
float scale = params[0];
|
||||
@@ -43,16 +93,19 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
ov::Output<ov::Node> res;
|
||||
|
||||
// For stateful
|
||||
std::string mask_name = "KQ_mask_sliced";
|
||||
if (context.get_input_names()[3].find("swa") != std::string::npos) {
|
||||
mask_name = "KQ_mask_swa_sliced";
|
||||
}
|
||||
if (context.has_input(mask_name)) {
|
||||
mask = context.get_input(mask_name);
|
||||
}
|
||||
|
||||
if (mask.get_element_type() != ov::element::f16) {
|
||||
mask = std::make_shared<ov::op::v0::Convert>(mask, ov::element::f16);
|
||||
ov::Output<ov::Node> mask;
|
||||
if (has_mask) {
|
||||
mask = context.get_input(3);
|
||||
std::string mask_name = "KQ_mask_sliced";
|
||||
if (context.get_input_names()[3].find("swa") != std::string::npos) {
|
||||
mask_name = "KQ_mask_swa_sliced";
|
||||
}
|
||||
if (context.has_input(mask_name)) {
|
||||
mask = context.get_input(mask_name);
|
||||
}
|
||||
if (mask.get_element_type() != ov::element::f16) {
|
||||
mask = std::make_shared<ov::op::v0::Convert>(mask, ov::element::f16);
|
||||
}
|
||||
}
|
||||
|
||||
//auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output<Node> kv) {
|
||||
@@ -108,10 +161,14 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
// get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the
|
||||
// [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and
|
||||
// 1→factor on the head dims.
|
||||
auto mask_unsq1 =
|
||||
std::make_shared<ov::op::v0::Unsqueeze>(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
|
||||
// mask_unsq1: [B, 1, 1, S_q, S_k] (rank 5)
|
||||
ov::Output<ov::Node> qk_masked = std::make_shared<ov::op::v1::Add>(qk_scaled, mask_unsq1);
|
||||
ov::Output<ov::Node> qk_masked;
|
||||
if (has_mask) {
|
||||
auto mask_unsq1 =
|
||||
std::make_shared<ov::op::v0::Unsqueeze>(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
|
||||
qk_masked = std::make_shared<ov::op::v1::Add>(qk_scaled, mask_unsq1);
|
||||
} else {
|
||||
qk_masked = qk_scaled;
|
||||
}
|
||||
|
||||
auto softmax = std::make_shared<ov::op::v8::Softmax>(qk_masked, /*axis=*/-1);
|
||||
|
||||
@@ -164,9 +221,16 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
k = tile_kv(num_heads, num_heads_kv, head_size, k);
|
||||
v = tile_kv(num_heads, num_heads_kv, head_size, v);
|
||||
|
||||
auto sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, mask, scale_node, false);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(sdpa,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
|
||||
constexpr auto causal = false;
|
||||
if (has_mask) {
|
||||
auto sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, mask, scale_node, causal);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(
|
||||
sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
|
||||
} else {
|
||||
auto sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, scale_node, causal);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(
|
||||
sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
|
||||
}
|
||||
res = std::make_shared<ov::op::v0::Convert>(res, ov::element::f32);
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
@@ -7,12 +7,15 @@
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/broadcast.hpp>
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/exp.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/less.hpp>
|
||||
#include <openvino/op/loop.hpp>
|
||||
#include <openvino/op/matmul.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
@@ -80,6 +83,28 @@ OutputVector translate_gated_delta_net(const NodeContext & context) {
|
||||
g = std::make_shared<ov::op::v0::Squeeze>(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
|
||||
beta = std::make_shared<ov::op::v0::Squeeze>(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
|
||||
|
||||
if (context.has_input("chunk_valid_len")) {
|
||||
// The last prefill chunk is padded with fabricated tokens. The recurrence is
|
||||
// S_t = S_{t-1} * exp(g_t) + k_t (x) ((v_t - S_{t-1}^T k_t) * beta_t)
|
||||
// so forcing g = 0 and beta = 0 makes a padded step an exact identity and keeps the final
|
||||
// state equal to the state after the last real token. Attention output at those positions
|
||||
// is garbage but never read.
|
||||
const auto & g_ps = g.get_partial_shape();
|
||||
FRONT_END_OP_CONVERSION_CHECK(g_ps.rank().is_static() && g_ps.rank().get_length() == 3 && g_ps[1].is_static(),
|
||||
"GATED_DELTA_NET pad masking requires a static token dimension");
|
||||
const int64_t n_tokens = g_ps[1].get_length();
|
||||
std::vector<int64_t> positions(n_tokens);
|
||||
std::iota(positions.begin(), positions.end(), 0);
|
||||
auto valid = std::make_shared<ov::op::v1::Less>(
|
||||
ov::op::v0::Constant::create(ov::element::i64, {(size_t) n_tokens}, positions),
|
||||
context.get_input("chunk_valid_len"));
|
||||
auto mask = std::make_shared<ov::op::v0::Unsqueeze>(
|
||||
std::make_shared<ov::op::v0::Convert>(valid, g.get_element_type()),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {2}, std::vector<int64_t>{0, 2}));
|
||||
g = std::make_shared<ov::op::v1::Multiply>(g, mask);
|
||||
beta = std::make_shared<ov::op::v1::Multiply>(beta, mask);
|
||||
}
|
||||
|
||||
// std::cout << "GatedDeltaNet input shapes: q=" << q.get_partial_shape() << ", k=" << k.get_partial_shape()
|
||||
// << ", v=" << v.get_partial_shape() << ", g=" << g.get_partial_shape()
|
||||
// << ", beta=" << beta.get_partial_shape() << ", state=" << state.get_partial_shape() << std::endl;
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
|
||||
#include <memory>
|
||||
#include <openvino/core/node_output.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/sigmoid.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace op {
|
||||
|
||||
OutputVector translate_glu_geglu_quick(const NodeContext & context) {
|
||||
num_inputs_check(context, 1, 2);
|
||||
|
||||
ov::Output<ov::Node> src0;
|
||||
ov::Output<ov::Node> src1;
|
||||
if (context.get_input_size() == 2) {
|
||||
src0 = process_view_input_new(context, 0);
|
||||
src1 = process_view_input_new(context, 1);
|
||||
} else {
|
||||
// split along last axis, nc = ne[0] / 2
|
||||
auto combined = process_view_input_new(context, 0);
|
||||
auto combined_shape = combined.get_partial_shape();
|
||||
int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length();
|
||||
int64_t nc = last_dim_val / 2;
|
||||
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
|
||||
auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
|
||||
auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
|
||||
auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
|
||||
auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});
|
||||
|
||||
src0 = std::make_shared<ov::op::v8::Slice>(combined, start0, stop0, step, axis);
|
||||
src1 = std::make_shared<ov::op::v8::Slice>(combined, start1, stop1, step, axis);
|
||||
}
|
||||
|
||||
int32_t * params = context.get_output_op_params();
|
||||
const int32_t swapped = params[1];
|
||||
if (swapped) {
|
||||
std::swap(src0, src1);
|
||||
}
|
||||
|
||||
// GELU_QUICK(x) = x * sigmoid(1.702 * x)
|
||||
// Create the constant in the same type as src0 to avoid f16/f32 mismatch.
|
||||
auto input_type = src0.get_element_type();
|
||||
auto coef = ov::op::v0::Constant::create(input_type, ov::Shape{}, {1.702f});
|
||||
auto scaled = std::make_shared<ov::op::v1::Multiply>(src0, coef);
|
||||
auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(scaled);
|
||||
auto gated = std::make_shared<ov::op::v1::Multiply>(src0, sigmoid);
|
||||
auto res = std::make_shared<ov::op::v1::Multiply>(gated, src1);
|
||||
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
} // namespace op
|
||||
} // namespace ggml
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
@@ -0,0 +1,53 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
|
||||
#include <openvino/op/avg_pool.hpp>
|
||||
#include <openvino/op/max_pool.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace op {
|
||||
|
||||
OutputVector translate_pool_2d(const NodeContext & context) {
|
||||
num_inputs_check(context, 1, 1);
|
||||
const int32_t * params = context.get_output_op_params();
|
||||
|
||||
const int k0 = params[1];
|
||||
const int k1 = params[2];
|
||||
const int s0 = params[3];
|
||||
const int s1 = params[4];
|
||||
const int p0 = params[5];
|
||||
const int p1 = params[6];
|
||||
|
||||
const int op_case = context.get_op_case();
|
||||
ov::Output<Node> input = context.get_input(0);
|
||||
ov::Strides strides{static_cast<size_t>(s1), static_cast<size_t>(s0)};
|
||||
ov::Shape pads_begin{static_cast<size_t>(p1), static_cast<size_t>(p0)};
|
||||
ov::Shape pads_end{static_cast<size_t>(p1), static_cast<size_t>(p0)};
|
||||
ov::Shape kernel{static_cast<size_t>(k1), static_cast<size_t>(k0)};
|
||||
ov::Output<Node> res;
|
||||
|
||||
switch (op_case) {
|
||||
case 1: // GGML_OP_POOL_MAX
|
||||
{
|
||||
res = std::make_shared<ov::op::v1::MaxPool>(input, strides, pads_begin, pads_end, kernel);
|
||||
break;
|
||||
}
|
||||
case 2: // GGML_OP_POOL_AVG
|
||||
{
|
||||
res = std::make_shared<ov::op::v1::AvgPool>(input, strides, pads_begin, pads_end, kernel, false);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
} // namespace op
|
||||
} // namespace ggml
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
@@ -0,0 +1,36 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
|
||||
#include <openvino/op/roll.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace op {
|
||||
|
||||
OutputVector translate_roll(const NodeContext & context) {
|
||||
num_inputs_check(context, 1, 1);
|
||||
const int32_t * params = context.get_output_op_params();
|
||||
|
||||
int64_t s0 = params[0];
|
||||
int64_t s1 = params[1];
|
||||
int64_t s2 = params[2];
|
||||
int64_t s3 = params[3];
|
||||
|
||||
auto input = context.get_input(0);
|
||||
|
||||
auto shift = ov::op::v0::Constant::create(
|
||||
ov::element::i64, ov::Shape{4}, std::vector<int64_t>{s3, s2, s1, s0});
|
||||
auto axes = ov::op::v0::Constant::create(
|
||||
ov::element::i64, ov::Shape{4}, std::vector<int64_t>{0, 1, 2, 3});
|
||||
|
||||
auto roll = std::make_shared<ov::op::v7::Roll>(input, shift, axes);
|
||||
return rename_outputs_with_suffix({roll}, context.get_name());
|
||||
}
|
||||
|
||||
} // namespace op
|
||||
} // namespace ggml
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
@@ -17,6 +17,13 @@ namespace op {
|
||||
OutputVector translate_view(const NodeContext & context) {
|
||||
num_inputs_check(context, 1, 1);
|
||||
|
||||
if (context.get_op_case() == 1) {
|
||||
// Static-mode identity pass-through for VIEWs over a GATED_DELTA_NET combined output or
|
||||
// the conv_input CONCAT; the consuming op (CPY/RMS_NORM) does its own runtime-correct
|
||||
// slicing on the full tensor (see ggml-decoder.cpp compute_op_case, GGML_OP_VIEW).
|
||||
return {context.get_input(0)};
|
||||
}
|
||||
|
||||
if (!context.is_static()) {
|
||||
// On the stateless/non-static path VIEW is normally a no-op (consumers re-slice).
|
||||
// EXCEPTION: the MoE expert aggregation slices each expert plane out of
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <openvino/op/matmul.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/negative.hpp>
|
||||
#include <openvino/op/relu.hpp>
|
||||
#include <openvino/op/sigmoid.hpp>
|
||||
#include <openvino/op/subtract.hpp>
|
||||
#include <openvino/op/tanh.hpp>
|
||||
@@ -55,10 +56,12 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
|
||||
{"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input<v0::Sigmoid> },
|
||||
{"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input<v0::Exp> },
|
||||
{"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input<v0::Negative> },
|
||||
{"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input<v0::Relu> },
|
||||
{"GGML_OP_VIEW", op::translate_view },
|
||||
{"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu },
|
||||
{"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai },
|
||||
{"GGML_GLU_OP_GEGLU", op::translate_glu_geglu },
|
||||
{"GGML_GLU_OP_GEGLU_QUICK", op::translate_glu_geglu_quick },
|
||||
{"GGML_OP_SET_ROWS", op::translate_set_rows },
|
||||
{"GGML_OP_CPY", op::translate_cpy },
|
||||
{"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext },
|
||||
@@ -72,6 +75,8 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
|
||||
{"GGML_OP_DIAG", op::translate_diag },
|
||||
{"GGML_OP_TRI", op::translate_tri },
|
||||
{"GGML_OP_SET", op::translate_set },
|
||||
{"GGML_OP_POOL_2D", op::translate_pool_2d },
|
||||
{"GGML_OP_ROLL", op::translate_roll },
|
||||
// solve_tri has accuracy issues on GPU
|
||||
// {"GGML_OP_SOLVE_TRI", op::translate_solve_tri },
|
||||
};
|
||||
|
||||
@@ -38,6 +38,7 @@ GGML_OP_CONVERTER(translate_view);
|
||||
GGML_OP_CONVERTER(translate_glu_swiglu);
|
||||
GGML_OP_CONVERTER(translate_glu_swiglu_oai);
|
||||
GGML_OP_CONVERTER(translate_glu_geglu);
|
||||
GGML_OP_CONVERTER(translate_glu_geglu_quick);
|
||||
GGML_OP_CONVERTER(translate_set_rows);
|
||||
GGML_OP_CONVERTER(translate_cpy);
|
||||
GGML_OP_CONVERTER(translate_argsort);
|
||||
@@ -53,6 +54,8 @@ GGML_OP_CONVERTER(translate_set);
|
||||
GGML_OP_CONVERTER(translate_diag);
|
||||
GGML_OP_CONVERTER(translate_tri);
|
||||
GGML_OP_CONVERTER(translate_solve_tri);
|
||||
GGML_OP_CONVERTER(translate_pool_2d);
|
||||
GGML_OP_CONVERTER(translate_roll);
|
||||
|
||||
} // namespace op
|
||||
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
#include "fuse_to_conv.h"
|
||||
|
||||
#include <openvino/core/graph_util.hpp>
|
||||
#include <openvino/core/rt_info.hpp>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/convolution.hpp>
|
||||
#include <openvino/op/extractimagepatches.hpp>
|
||||
#include <openvino/op/matmul.hpp>
|
||||
#include <openvino/op/pad.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/transpose.hpp>
|
||||
#include <openvino/pass/pattern/op/label.hpp>
|
||||
#include <openvino/pass/pattern/op/pattern.hpp>
|
||||
#include <openvino/pass/pattern/op/wrap_type.hpp>
|
||||
|
||||
namespace opp = ov::pass::pattern;
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace pass {
|
||||
|
||||
// This pass fuses an IM2COL + MatMul convolution into OpenVINO's Convolution op for performance gains.
|
||||
// Reference the im2col.cpp translator for reference on the pattern being matched.
|
||||
|
||||
FuseToConv::FuseToConv() {
|
||||
const auto m_wei = opp::any_input();
|
||||
const auto m_act = opp::any_input();
|
||||
const auto m_matmul = opp::wrap_type<ov::op::v0::MatMul>({m_wei, m_act});
|
||||
|
||||
const auto callback = [=](ov::pass::pattern::Matcher & m) {
|
||||
const auto & pm = m.get_pattern_value_map();
|
||||
|
||||
auto matmul_node = ov::as_type_ptr<ov::op::v0::MatMul>(pm.at(m_matmul).get_node_shared_ptr());
|
||||
if (!matmul_node || matmul_node->get_transpose_a() || !matmul_node->get_transpose_b()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto trace = matmul_node->input_value(1);
|
||||
|
||||
// Optional Convert
|
||||
if (auto n = ov::as_type_ptr<ov::op::v0::Convert>(trace.get_node_shared_ptr())) {
|
||||
trace = n->input_value(0);
|
||||
}
|
||||
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
auto n = ov::as_type_ptr<ov::op::v1::Reshape>(trace.get_node_shared_ptr());
|
||||
if (!n) {
|
||||
return false;
|
||||
}
|
||||
trace = n->input_value(0);
|
||||
}
|
||||
|
||||
if (auto n = ov::as_type_ptr<ov::op::v1::Transpose>(trace.get_node_shared_ptr())) {
|
||||
trace = n->input_value(0);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (auto n = ov::as_type_ptr<ov::op::v1::Reshape>(trace.get_node_shared_ptr())) {
|
||||
trace = n->input_value(0);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (auto n = ov::as_type_ptr<ov::op::v1::Transpose>(trace.get_node_shared_ptr())) {
|
||||
trace = n->input_value(0);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto eip = ov::as_type_ptr<ov::op::v3::ExtractImagePatches>(trace.get_node_shared_ptr());
|
||||
if (!eip) {
|
||||
return false;
|
||||
}
|
||||
const auto eip_strides = eip->get_strides(); // {stride_h, stride_w}
|
||||
const auto eip_rates = eip->get_rates(); // {dil_h, dil_w}
|
||||
|
||||
auto pad = ov::as_type_ptr<ov::op::v1::Pad>(eip->input_value(0).get_node_shared_ptr());
|
||||
if (!pad) {
|
||||
return false;
|
||||
}
|
||||
auto pads_begin_const =
|
||||
ov::as_type_ptr<ov::op::v0::Constant>(pad->input_value(1).get_node_shared_ptr());
|
||||
|
||||
const auto pads_begin_vals = pads_begin_const->cast_vector<int64_t>(); // {0, 0, pad_h, pad_w}
|
||||
const std::ptrdiff_t pad_h = static_cast<std::ptrdiff_t>(pads_begin_vals[2]);
|
||||
const std::ptrdiff_t pad_w = static_cast<std::ptrdiff_t>(pads_begin_vals[3]);
|
||||
|
||||
auto image_input = pad->input_value(0); // [N, IC, 1, IW] NCHW
|
||||
|
||||
auto w_trace = matmul_node->input_value(0);
|
||||
if (auto n = ov::as_type_ptr<ov::op::v0::Convert>(w_trace.get_node_shared_ptr())) {
|
||||
w_trace = n->input_value(0);
|
||||
}
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
auto n = ov::as_type_ptr<ov::op::v1::Reshape>(w_trace.get_node_shared_ptr());
|
||||
if (!n) {
|
||||
break;
|
||||
}
|
||||
w_trace = n->input_value(0);
|
||||
}
|
||||
|
||||
auto weight_const = ov::as_type_ptr<ov::op::v0::Constant>(w_trace.get_node_shared_ptr());
|
||||
if (!weight_const) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Reshape weight to [OC, IC, 1, KW] (OIHW).
|
||||
const auto w_shape = weight_const->get_shape();
|
||||
ov::Shape conv_w_shape;
|
||||
if (w_shape.size() == 3) {
|
||||
conv_w_shape = {w_shape[0], w_shape[1], 1, w_shape[2]};
|
||||
} else if (w_shape.size() == 4) {
|
||||
conv_w_shape = {w_shape[1], w_shape[2], 1, w_shape[3]};
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto weight_reshaped = register_new_node<ov::op::v0::Constant>(weight_const->get_element_type(), conv_w_shape,
|
||||
weight_const->get_data_ptr());
|
||||
|
||||
ov::Output<Node> weight_input = weight_reshaped;
|
||||
if (weight_reshaped->get_element_type() != image_input.get_element_type()) {
|
||||
weight_input = register_new_node<ov::op::v0::Convert>(weight_reshaped, image_input.get_element_type());
|
||||
}
|
||||
|
||||
auto conv = register_new_node<ov::op::v1::Convolution>(
|
||||
image_input, weight_input,
|
||||
ov::Strides{static_cast<size_t>(eip_strides[0]), static_cast<size_t>(eip_strides[1])},
|
||||
ov::CoordinateDiff{pad_h, pad_w}, ov::CoordinateDiff{pad_h, pad_w},
|
||||
ov::Strides{static_cast<size_t>(eip_rates[0]), static_cast<size_t>(eip_rates[1])},
|
||||
ov::op::PadType::EXPLICIT);
|
||||
|
||||
constexpr auto target_type = ov::element::f32;
|
||||
ov::Output<Node> conv_out = conv;
|
||||
if (conv_out.get_element_type() != target_type) {
|
||||
conv_out = register_new_node<ov::op::v0::Convert>(conv_out, target_type);
|
||||
}
|
||||
|
||||
std::shared_ptr<ov::op::v1::Add> add_node;
|
||||
ov::Output<Node> bias_input;
|
||||
for (const auto & consumer_in : matmul_node->output(0).get_target_inputs()) {
|
||||
auto cast = ov::as_type_ptr<ov::op::v0::Convert>(consumer_in.get_node()->shared_from_this());
|
||||
if (!cast) {
|
||||
continue;
|
||||
}
|
||||
for (const auto & add_in : cast->output(0).get_target_inputs()) {
|
||||
auto add = ov::as_type_ptr<ov::op::v1::Add>(add_in.get_node()->shared_from_this());
|
||||
if (!add) {
|
||||
continue;
|
||||
}
|
||||
for (size_t i = 0; i < 2; ++i) {
|
||||
if (ov::as_type_ptr<ov::op::v0::Constant>(add->input_value(i).get_node_shared_ptr())) {
|
||||
bias_input = add->input_value(i);
|
||||
add_node = add;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (add_node) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (add_node) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ov::Output<Node> final_out;
|
||||
std::shared_ptr<Node> target_node;
|
||||
|
||||
if (add_node) {
|
||||
// Reshape bias [OC, 1] → [1, OC, 1, 1] for NCHW broadcasting.
|
||||
ov::Output<Node> bias = bias_input;
|
||||
if (bias.get_element_type() != target_type) {
|
||||
bias = register_new_node<ov::op::v0::Convert>(bias, target_type);
|
||||
}
|
||||
const auto oc = static_cast<int64_t>(conv_w_shape[0]);
|
||||
auto bias_shape = register_new_node<ov::op::v0::Constant>(ov::element::i64, ov::Shape{4},
|
||||
std::vector<int64_t>{1, oc, 1, 1});
|
||||
bias = register_new_node<ov::op::v1::Reshape>(bias, bias_shape, false);
|
||||
final_out = register_new_node<ov::op::v1::Add>(conv_out, bias);
|
||||
target_node = add_node;
|
||||
} else {
|
||||
final_out = conv_out;
|
||||
target_node = matmul_node;
|
||||
}
|
||||
|
||||
// Reshape final output back to the target node's original shape if needed.
|
||||
auto orig_shape = target_node->get_output_partial_shape(0);
|
||||
if (orig_shape.is_static() && final_out.get_partial_shape() != orig_shape) {
|
||||
auto shape_const = register_new_node<ov::op::v0::Constant>(ov::element::i64, ov::Shape{orig_shape.size()},
|
||||
orig_shape.to_shape());
|
||||
final_out = register_new_node<ov::op::v1::Reshape>(final_out, shape_const, false);
|
||||
}
|
||||
|
||||
final_out.get_node_shared_ptr()->set_friendly_name(target_node->get_friendly_name());
|
||||
ov::copy_runtime_info(m.get_matched_nodes(), final_out.get_node_shared_ptr());
|
||||
ov::replace_node(target_node, final_out.get_node_shared_ptr());
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
register_matcher(std::make_shared<opp::Matcher>(m_matmul, "ov::frontend::ggml::pass::FuseToConv"), callback);
|
||||
}
|
||||
|
||||
} // namespace pass
|
||||
} // namespace ggml
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
@@ -0,0 +1,17 @@
|
||||
#include "openvino/pass/matcher_pass.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace pass {
|
||||
|
||||
class FuseToConv : public ov::pass::MatcherPass {
|
||||
public:
|
||||
OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseToConv")
|
||||
FuseToConv();
|
||||
};
|
||||
|
||||
} // namespace pass
|
||||
} // namespace ggml
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "ggml-openvino/openvino/node_context.h"
|
||||
#include "ggml-openvino/openvino/utils.h"
|
||||
#include "input_model.h"
|
||||
#include "pass/fuse_to_conv.h"
|
||||
#include "pass/mark_decompression_convert_constant_folding.h"
|
||||
#include "pass/mark_dequantization_subgraph.h"
|
||||
#include "pass/squeeze_matmul.h"
|
||||
@@ -109,7 +110,8 @@ ov::pass::MakeStateful::ParamResPairs get_kv_param_res_pairs(
|
||||
void add_sliced_mask_stateful(TensorMap & tensor_map) {
|
||||
auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name) {
|
||||
if ((tensor_map.find(mask_name) != tensor_map.end()) &&
|
||||
(tensor_map.find("token_len_per_seq") != tensor_map.end())) {
|
||||
(tensor_map.find("token_len_per_seq") != tensor_map.end()) &&
|
||||
(tensor_map.find("inp_pos") != tensor_map.end())) {
|
||||
auto token_len_per_seq = tensor_map.at("token_len_per_seq").get_node_shared_ptr();
|
||||
auto mask = tensor_map.at(mask_name).get_node_shared_ptr();
|
||||
std::shared_ptr<ov::Node> mask_sliced = mask;
|
||||
@@ -137,6 +139,7 @@ void add_sliced_mask_stateful(TensorMap & tensor_map) {
|
||||
};
|
||||
|
||||
create_sliced_mask("self_kq_mask", "KQ_mask_sliced");
|
||||
create_sliced_mask("KQ_mask", "KQ_mask_sliced");
|
||||
create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced");
|
||||
}
|
||||
|
||||
@@ -395,6 +398,7 @@ std::shared_ptr<Model> TranslateSession::apply_transformations(std::shared_ptr<M
|
||||
// is_decompression_multiply() recognizes GatherMatmul as a valid consumer.
|
||||
manager.register_pass<ov::pass::MarkDequantization>(
|
||||
std::vector<ov::element::Type>{ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4});
|
||||
manager.register_pass<pass::FuseToConv>();
|
||||
|
||||
if (ggml_model_decoder->is_stateful()) {
|
||||
const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names();
|
||||
|
||||
@@ -72,6 +72,7 @@ OutputVector rename_outputs_with_suffix(const OutputVector & outputs, const std:
|
||||
name += "_";
|
||||
name += suffix;
|
||||
node->set_friendly_name(name);
|
||||
// Uncomment to dump every node's inferred shape (used to hunt down dynamic dims on NPU).
|
||||
// std::cout << name << " " << output.get_partial_shape() << std::endl;
|
||||
}
|
||||
return outputs;
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
@@ -48,7 +50,7 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend)
|
||||
GgmlOvDecoder::dump_cgraph(cgraph, filename);
|
||||
}
|
||||
|
||||
const auto is_static = ggml_openvino_is_npu();
|
||||
const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC");
|
||||
|
||||
GGML_ASSERT(ctx->runtime_context != nullptr);
|
||||
std::shared_ptr<ov_runtime_context> r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context);
|
||||
@@ -168,13 +170,24 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
|
||||
|
||||
auto output_type = ggml_decoder->get_ov_type(ggml_tensor);
|
||||
ov::Shape output_shape;
|
||||
void * output_data = ggml_tensor->data;
|
||||
if (ggml_decoder->is_static()) {
|
||||
output_shape = infer_request->get_output_tensor(output_index).get_shape();
|
||||
} else {
|
||||
output_shape = ggml_decoder->get_shape(ggml_tensor);
|
||||
// For a CPY into a padded view_src (e.g. a padded KV cache buffer), the
|
||||
// OV ScatterUpdate node outputs the full view_src shape, not the CPY node's
|
||||
// own (smaller) shape. Using the CPY shape here causes set_output_tensor to
|
||||
// fail with a shape-incompatibility error. Use view_src's shape and data
|
||||
// pointer instead so the OV tensor matches the model output exactly.
|
||||
if (ggml_tensor->op == GGML_OP_CPY && ggml_tensor->view_src != nullptr &&
|
||||
ggml_nbytes(ggml_tensor) != ggml_nbytes(ggml_tensor->view_src)) {
|
||||
output_shape = ggml_decoder->get_shape(ggml_tensor->view_src);
|
||||
output_data = ggml_tensor->view_src->data;
|
||||
} else {
|
||||
output_shape = ggml_decoder->get_shape(ggml_tensor);
|
||||
}
|
||||
}
|
||||
|
||||
ov::Tensor output_tensor(output_type, output_shape, ggml_tensor->data);
|
||||
ov::Tensor output_tensor(output_type, output_shape, output_data);
|
||||
return output_tensor;
|
||||
}
|
||||
|
||||
@@ -583,7 +596,9 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
|
||||
return chunk_size;
|
||||
};
|
||||
|
||||
static std::string device = "NPU";
|
||||
// Normally NPU, but honors GGML_OPENVINO_DEVICE so GGML_OPENVINO_FORCE_STATIC can run the
|
||||
// static-shape path on CPU/GPU to isolate translation bugs from NPUW/NPU-driver issues.
|
||||
static std::string device = ggml_openvino_get_device_name();
|
||||
static auto is_static = true;
|
||||
static auto stateful = false;
|
||||
|
||||
@@ -603,7 +618,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
|
||||
std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static);
|
||||
|
||||
const auto * inp_pos = get_inp_pos_tensor(cgraph);
|
||||
const auto is_prefill = get_is_prefill(inp_pos);
|
||||
const auto is_prefill = get_is_prefill(cgraph, inp_pos);
|
||||
graph_key key(cgraph);
|
||||
static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
|
||||
bool cache_hit = false;
|
||||
@@ -687,38 +702,55 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
|
||||
stateful, false, false, prefill_chunk_size);
|
||||
decoder_end_time = ggml_time_us();
|
||||
|
||||
auto input_model_prefill = std::make_shared<ov::frontend::ggml::InputModel>(ggml_decoder_prefill);
|
||||
auto input_model_decode = std::make_shared<ov::frontend::ggml::InputModel>(ggml_decoder_decode);
|
||||
const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR");
|
||||
const auto dump_ir_timestamp = static_cast<long long>(ggml_time_us());
|
||||
|
||||
auto model_prefill = ov::frontend::ggml::FrontEnd::convert(input_model_prefill);
|
||||
ggml_decoder_prefill->clear_model_weights();
|
||||
auto model_decode = ov::frontend::ggml::FrontEnd::convert(input_model_decode);
|
||||
ggml_decoder_decode->clear_model_weights();
|
||||
conversion_end_time = ggml_time_us();
|
||||
auto build_static_model = [&core, &config, dump_ir, dump_ir_timestamp](
|
||||
std::shared_ptr<GgmlOvDecoder> decoder,
|
||||
const char * tag,
|
||||
std::shared_ptr<ov::Model> & model,
|
||||
ov::CompiledModel & compiled_model,
|
||||
std::shared_ptr<ov::InferRequest> & infer_request,
|
||||
int64_t & local_conversion_end_time,
|
||||
int64_t & local_compile_end_time) {
|
||||
auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder);
|
||||
model = ov::frontend::ggml::FrontEnd::convert(input_model);
|
||||
decoder->clear_model_weights();
|
||||
local_conversion_end_time = ggml_time_us();
|
||||
|
||||
if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
|
||||
char timestamped_filename[64];
|
||||
auto timestamp = (long long) ggml_time_us();
|
||||
snprintf(timestamped_filename, sizeof(timestamped_filename), "model_prefill_%lld.xml", timestamp);
|
||||
ov::serialize(model_prefill, timestamped_filename);
|
||||
snprintf(timestamped_filename, sizeof(timestamped_filename), "model_decode_%lld.xml", timestamp);
|
||||
ov::serialize(model_decode, timestamped_filename);
|
||||
}
|
||||
if (dump_ir) {
|
||||
char timestamped_filename[64];
|
||||
snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag,
|
||||
dump_ir_timestamp);
|
||||
ov::serialize(model, timestamped_filename);
|
||||
}
|
||||
|
||||
compiled_model = core.compile_model(model, device, config);
|
||||
infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
|
||||
local_compile_end_time = ggml_time_us();
|
||||
};
|
||||
std::shared_ptr<ov::Model> model_prefill;
|
||||
std::shared_ptr<ov::Model> model_decode;
|
||||
ov::CompiledModel compiled_model_prefill;
|
||||
ov::CompiledModel compiled_model_decode;
|
||||
auto remote_context = ggml_openvino_get_remote_context();
|
||||
if (remote_context.has_value()) {
|
||||
compiled_model_prefill = core.compile_model(model_prefill, remote_context.value(), config);
|
||||
compiled_model_decode = core.compile_model(model_decode, remote_context.value(), config);
|
||||
} else {
|
||||
compiled_model_prefill = core.compile_model(model_prefill, device, config);
|
||||
compiled_model_decode = core.compile_model(model_decode, device, config);
|
||||
}
|
||||
|
||||
auto infer_request_prefill = std::make_shared<ov::InferRequest>(compiled_model_prefill.create_infer_request());
|
||||
auto infer_request_decode = std::make_shared<ov::InferRequest>(compiled_model_decode.create_infer_request());
|
||||
compile_end_time = ggml_time_us();
|
||||
std::shared_ptr<ov::InferRequest> infer_request_prefill;
|
||||
std::shared_ptr<ov::InferRequest> infer_request_decode;
|
||||
int64_t prefill_conversion_end_time;
|
||||
int64_t decode_conversion_end_time;
|
||||
int64_t prefill_compile_end_time;
|
||||
int64_t decode_compile_end_time;
|
||||
auto prefill_future = std::async(std::launch::async, build_static_model, ggml_decoder_prefill, "prefill",
|
||||
std::ref(model_prefill), std::ref(compiled_model_prefill),
|
||||
std::ref(infer_request_prefill), std::ref(prefill_conversion_end_time),
|
||||
std::ref(prefill_compile_end_time));
|
||||
auto decode_future = std::async(std::launch::async, build_static_model, ggml_decoder_decode, "decode",
|
||||
std::ref(model_decode), std::ref(compiled_model_decode),
|
||||
std::ref(infer_request_decode), std::ref(decode_conversion_end_time),
|
||||
std::ref(decode_compile_end_time));
|
||||
prefill_future.get();
|
||||
decode_future.get();
|
||||
conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time);
|
||||
compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time);
|
||||
|
||||
model = is_prefill ? model_prefill : model_decode;
|
||||
ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode;
|
||||
@@ -742,7 +774,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
|
||||
}
|
||||
|
||||
if (is_prefill) {
|
||||
auto inp_len = inp_pos->ne[0];
|
||||
auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos);
|
||||
for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) {
|
||||
for (size_t i = 0; i < ov_input_names_local.size(); i++) {
|
||||
auto param_name = ov_input_names_local[i];
|
||||
@@ -762,6 +794,11 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
|
||||
continue;
|
||||
}
|
||||
auto * ggml_tensor = model_output_it->second;
|
||||
if (ggml_nbytes(ggml_tensor) == 0) {
|
||||
// Zero-row in-place writeback (e.g. the empty s_copy defrag remainder). The OV
|
||||
// Result is the full cache, so binding it over this 0-byte buffer overflows it.
|
||||
continue;
|
||||
}
|
||||
auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
|
||||
infer_request->set_output_tensor(i, output_tensor);
|
||||
}
|
||||
@@ -798,6 +835,9 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
|
||||
continue;
|
||||
}
|
||||
auto * ggml_tensor = model_output_it->second;
|
||||
if (ggml_nbytes(ggml_tensor) == 0) {
|
||||
continue;
|
||||
}
|
||||
auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
|
||||
infer_request->set_output_tensor(i, output_tensor);
|
||||
}
|
||||
@@ -1074,6 +1114,9 @@ ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, cons
|
||||
ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
|
||||
const std::string & param_name) {
|
||||
// NPU decoding stage
|
||||
if (ggml_decoder->get_model_extra_inputs().count(param_name)) {
|
||||
return get_ov_input_tensor(ggml_decoder, param_name);
|
||||
}
|
||||
const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);
|
||||
const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);
|
||||
|
||||
@@ -1123,14 +1166,30 @@ ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggm
|
||||
const std::string & param_name,
|
||||
int chunk_index) {
|
||||
// NPU prompt processing stage
|
||||
const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);
|
||||
const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);
|
||||
|
||||
const size_t input_len = ggml_decoder->get_input_len();
|
||||
const size_t chunk_size = ggml_decoder->m_prefill_chunk_size;
|
||||
const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size);
|
||||
const size_t chunk_pad_size = chunk_size - chunk_valid_size;
|
||||
|
||||
if (param_name == "chunk_valid_len") {
|
||||
ov::Tensor input_tensor(ov::element::i64, ov::Shape{1});
|
||||
*input_tensor.data<int64_t>() = (int64_t) chunk_valid_size;
|
||||
return input_tensor;
|
||||
}
|
||||
if (chunk_index > 0 && param_name == "cache_rs_reset_len") {
|
||||
// The recurrent-state clear belongs to the start of the sequence. Re-applying it on every
|
||||
// chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero
|
||||
// length makes scale.cpp's keep-mask select every slot) after the first chunk.
|
||||
ov::Tensor input_tensor(ov::element::i64, ov::Shape{1});
|
||||
*input_tensor.data<int64_t>() = 0;
|
||||
return input_tensor;
|
||||
}
|
||||
if (ggml_decoder->get_model_extra_inputs().count(param_name)) {
|
||||
return get_ov_input_tensor(ggml_decoder, param_name);
|
||||
}
|
||||
const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);
|
||||
const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);
|
||||
|
||||
if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) {
|
||||
// IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length
|
||||
// input_len; pad every plane independently so they stay aligned to chunk_size.
|
||||
@@ -1306,7 +1365,7 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor
|
||||
<< std::endl;
|
||||
switch (tensor.get_element_type()) {
|
||||
case ov::element::f32: {
|
||||
if (name.find("self_kq_mask") == std::string::npos) {
|
||||
if (name.find("self_kq_mask") == std::string::npos && name.find("KQ_mask") == std::string::npos) {
|
||||
std::cout << *(tensor.data<float>()) << std::endl;
|
||||
} else {
|
||||
size_t rows = tensor.get_shape()[2];
|
||||
@@ -1414,8 +1473,24 @@ const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) {
|
||||
throw std::runtime_error("get_inp_pos_tensor: inp_pos not found in cgraph");
|
||||
}
|
||||
|
||||
bool get_is_prefill(const ggml_tensor * inp_pos) {
|
||||
return inp_pos->ne[0] > 1;
|
||||
int64_t get_inp_pos_n_tokens(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) {
|
||||
// IMROPE stacks n_planes (t/h/w/e) position planes into inp_pos, so ne[0] is
|
||||
// n_planes * n_tokens. Callers that need a token count must divide the planes out.
|
||||
int n_planes = 1;
|
||||
for (int i = 0; i < cgraph->n_nodes; ++i) {
|
||||
auto * op = cgraph->nodes[i];
|
||||
for (int j = 0; j < GGML_MAX_SRC; ++j) {
|
||||
if (op->src[j] == inp_pos) {
|
||||
n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return inp_pos->ne[0] / n_planes;
|
||||
}
|
||||
|
||||
bool get_is_prefill(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) {
|
||||
return get_inp_pos_n_tokens(cgraph, inp_pos) > 1;
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
@@ -164,7 +164,9 @@ std::vector<T> pad_input(const ggml_tensor * tensor, size_t padded_rows, size_t
|
||||
|
||||
const ggml_tensor * get_inp_pos_tensor(struct ggml_cgraph * cgraph);
|
||||
|
||||
bool get_is_prefill(const ggml_tensor * inp_pos);
|
||||
int64_t get_inp_pos_n_tokens(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos);
|
||||
|
||||
bool get_is_prefill(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos);
|
||||
|
||||
ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name);
|
||||
ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
|
||||
|
||||
@@ -1348,6 +1348,8 @@ struct vk_mat_mat_id_push_constants {
|
||||
uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;
|
||||
uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;
|
||||
uint32_t padded_N;
|
||||
uint32_t n_experts;
|
||||
uint32_t hoist_row_ids;
|
||||
};
|
||||
struct vk_mat_vec_id_push_constants {
|
||||
uint32_t ncols;
|
||||
@@ -1428,6 +1430,10 @@ struct vk_op_count_experts_push_constants {
|
||||
uint32_t nb00;
|
||||
uint32_t nb01;
|
||||
uint32_t a_offset;
|
||||
uint32_t n_experts;
|
||||
uint32_t hoist_row_ids;
|
||||
uint32_t ne00mp;
|
||||
uint32_t ne00L;
|
||||
};
|
||||
|
||||
struct vk_op_glu_push_constants {
|
||||
@@ -1606,6 +1612,10 @@ template <> void init_pushconst_fastdiv(vk_op_glu_push_constants &p) {
|
||||
init_fastdiv_values(p.ne20, p.ne2_0mp, p.ne2_0L);
|
||||
}
|
||||
|
||||
template <> void init_pushconst_fastdiv(vk_op_count_experts_push_constants &p) {
|
||||
init_fastdiv_values(p.ne00, p.ne00mp, p.ne00L);
|
||||
}
|
||||
|
||||
struct vk_op_binary_push_constants {
|
||||
uint32_t ne;
|
||||
uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
|
||||
@@ -5839,7 +5849,11 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_equal_i32, "count_equal_i32", count_equal_i32_len, count_equal_i32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, { device->subgroup_size }, 1);
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true);
|
||||
if (device->subgroup_arithmetic && device->subgroup_require_full_support) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true);
|
||||
} else {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true);
|
||||
}
|
||||
|
||||
for (auto &s : device->pipeline_solve_tri_f32) {
|
||||
const vk_solve_tri_pipeline_state &state = s.first;
|
||||
@@ -8970,13 +8984,13 @@ static void ggml_vk_matmul_id(
|
||||
uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d,
|
||||
uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d,
|
||||
uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11,
|
||||
uint32_t padded_n) {
|
||||
uint32_t padded_n, bool hoist_row_ids) {
|
||||
VK_LOG_DEBUG("ggml_vk_matmul_id(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), ids: (" << ids.buffer->buffer << ", " << ids.offset << ", " << ids.size << "), expert_count: (" << expert_count_buf.buffer->buffer << ", " << expert_count_buf.offset << ", " << expert_count_buf.size << "), " <<
|
||||
"m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", " <<
|
||||
"batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", " <<
|
||||
"n_as: " << n_as << ", nei0: " << nei0 << ", nei1: " << nei1 << ", nbi1: " << nbi1 << ", ne11: " << ne11 << ")");
|
||||
const vk_mat_mat_id_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d,
|
||||
nei0, nei1, nbi1, ne11, padded_n };
|
||||
nei0, nei1, nbi1, ne11, padded_n, n_as, uint32_t(hoist_row_ids) };
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, { m, nei1, n_as });
|
||||
}
|
||||
|
||||
@@ -10162,6 +10176,12 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
// const uint64_t ne23 = dst->ne[3];
|
||||
|
||||
const uint64_t n_as = ne02;
|
||||
// n_as counts, n_as offsets, one total, then one packed row id per (expert, token).
|
||||
// Hoisting requires 16-bit indices for the packing and a table that fits one binding.
|
||||
const uint64_t hoisted_row_id_words = 2 * n_as + 1 + nei0 * nei1;
|
||||
const bool hoist_row_ids = n_as <= 256 && nei0 <= 0xffff && nei1 <= 0xffff &&
|
||||
hoisted_row_id_words * sizeof(uint32_t) <=
|
||||
ctx->device->properties.limits.maxStorageBufferRange;
|
||||
|
||||
ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context;
|
||||
ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context;
|
||||
@@ -10302,7 +10322,8 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
}
|
||||
vk_pipeline count_experts = ctx->device->pipeline_count_experts;
|
||||
|
||||
uint32_t expert_count_size = sizeof(uint32_t) * n_as;
|
||||
const size_t expert_data_size = sizeof(uint32_t) *
|
||||
(hoist_row_ids ? hoisted_row_id_words : n_as);
|
||||
|
||||
{
|
||||
if (
|
||||
@@ -10318,8 +10339,8 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
ctx->prealloc_size_y = y_sz;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
if (ctx->prealloc_size_split_k < expert_count_size) {
|
||||
ctx->prealloc_size_split_k = expert_count_size;
|
||||
if (ctx->prealloc_size_split_k < expert_data_size) {
|
||||
ctx->prealloc_size_split_k = expert_data_size;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
|
||||
@@ -10385,18 +10406,23 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
}
|
||||
}
|
||||
// Count how many times each expert is used
|
||||
vk_subbuffer expert_count_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0);
|
||||
vk_subbuffer expert_count_buf = { ctx->prealloc_split_k, 0, expert_data_size };
|
||||
if (ctx->prealloc_split_k_need_sync) {
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
}
|
||||
{
|
||||
const std::vector<uint32_t> pc = { (uint32_t)nei0,
|
||||
vk_op_count_experts_push_constants pc = { (uint32_t)nei0,
|
||||
(uint32_t)nei1,
|
||||
(uint32_t)(nbi0 / ggml_type_size(ids->type)),
|
||||
(uint32_t)(nbi1 / ggml_type_size(ids->type)),
|
||||
(uint32_t)(get_misalign_bytes(ctx, ids) / ggml_type_size(ids->type)) };
|
||||
(uint32_t)(get_misalign_bytes(ctx, ids) / ggml_type_size(ids->type)),
|
||||
(uint32_t)n_as,
|
||||
uint32_t(hoist_row_ids),
|
||||
0, 0 };
|
||||
init_pushconst_fastdiv(pc);
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, count_experts,
|
||||
{ vk_subbuffer{ d_ids, ids_buf_offset, ids_sz }, expert_count_buf }, pc, { (uint32_t)n_as, 1, 1});
|
||||
{ vk_subbuffer{ d_ids, ids_buf_offset, ids_sz }, expert_count_buf }, pc,
|
||||
{ hoist_row_ids ? 1u : (uint32_t)n_as, 1, 1});
|
||||
}
|
||||
|
||||
if (x_non_contig) {
|
||||
@@ -10465,7 +10491,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
{ d_D, d_buf_offset, d_sz }, { d_ids, ids_buf_offset, ids_sz }, expert_count_buf,
|
||||
ne01, ne21, ne10, ne10, stride_b_y, ne01,
|
||||
stride_batch_x, stride_batch_y, ne20*ne21,
|
||||
n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, padded_n
|
||||
n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, padded_n, hoist_row_ids
|
||||
); // NOLINT
|
||||
|
||||
if (x_non_contig || qx_needs_dequant) {
|
||||
@@ -17774,20 +17800,32 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
|
||||
return;
|
||||
}
|
||||
|
||||
auto const &is_empty = [](ggml_tensor * node) -> bool {
|
||||
auto const &is_empty = [](const ggml_tensor * node) -> bool {
|
||||
return node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE;
|
||||
};
|
||||
|
||||
auto const &is_src_of = [](const ggml_tensor *dst, const ggml_tensor *src) -> bool {
|
||||
auto const &is_src_of = [&is_empty](const ggml_tensor *dst, const ggml_tensor *src) -> bool {
|
||||
auto const &base = [](const ggml_tensor * tensor) {
|
||||
return tensor->view_src ? tensor->view_src : tensor;
|
||||
};
|
||||
for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) {
|
||||
if (dst->src[s] == src) {
|
||||
return true;
|
||||
}
|
||||
if (is_empty(dst) || is_empty(src)) {
|
||||
continue;
|
||||
}
|
||||
// A source view of dst may read storage written through a different view by src.
|
||||
if (dst->src[s] && base(dst->src[s]) == base(src)) {
|
||||
return true;
|
||||
}
|
||||
// Moving dst forward may overwrite storage still read through a view by src.
|
||||
if (src->src[s] && base(dst) == base(src->src[s])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// implicit dependency if they view the same tensor
|
||||
const ggml_tensor *dst2 = dst->view_src ? dst->view_src : dst;
|
||||
const ggml_tensor *src2 = src->view_src ? src->view_src : src;
|
||||
if (dst2 == src2) {
|
||||
if (base(dst) == base(src)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
||||
@@ -2,6 +2,11 @@
|
||||
|
||||
#extension GL_EXT_control_flow_attributes : enable
|
||||
|
||||
#ifdef USE_SUBGROUPS
|
||||
#extension GL_KHR_shader_subgroup_basic : enable
|
||||
#extension GL_KHR_shader_subgroup_arithmetic : enable
|
||||
#endif
|
||||
|
||||
#include "types.glsl"
|
||||
|
||||
layout (push_constant) uniform parameter
|
||||
@@ -11,6 +16,10 @@ layout (push_constant) uniform parameter
|
||||
uint32_t nb00;
|
||||
uint32_t nb01;
|
||||
uint32_t a_offset;
|
||||
uint32_t n_experts;
|
||||
uint32_t hoist_row_ids;
|
||||
uint32_t ne00mp;
|
||||
uint32_t ne00L;
|
||||
} p;
|
||||
|
||||
#define BLOCK_SIZE 256
|
||||
@@ -21,16 +30,98 @@ layout (binding = 0) readonly buffer A {uint data_a[];};
|
||||
layout (binding = 1) writeonly buffer D {uint data_d[];};
|
||||
|
||||
shared uint vals[BLOCK_SIZE];
|
||||
shared uint offsets[BLOCK_SIZE];
|
||||
shared uint cursors[BLOCK_SIZE];
|
||||
|
||||
// see init_fastdiv_values in ggml-vulkan.cpp
|
||||
uint fastdiv(uint n, uint mp, uint L) {
|
||||
uint msbs, lsbs;
|
||||
// msbs = mulhi(n, mp)
|
||||
umulExtended(n, mp, msbs, lsbs);
|
||||
return (msbs + n) >> L;
|
||||
}
|
||||
|
||||
// data_d layout when p.hoist_row_ids is set:
|
||||
// [0, n_experts) per-expert row count
|
||||
// [n_experts, 2*n_experts) per-expert start offset into the row id region
|
||||
// [2*n_experts] total row count
|
||||
// [2*n_experts + 1, ) row ids grouped by expert, packed as (i01 << 16) | (i00 & 0xffff)
|
||||
// Otherwise only data_d[expert_id] is written, holding that expert's row count.
|
||||
void main() {
|
||||
const uint expert_id = gl_WorkGroupID.x;
|
||||
const uint num_elements = p.ne00 * p.ne01;
|
||||
const uint tid = gl_LocalInvocationID.x;
|
||||
|
||||
if (p.hoist_row_ids != 0) {
|
||||
if (tid < p.n_experts) {
|
||||
vals[tid] = 0;
|
||||
}
|
||||
barrier();
|
||||
|
||||
for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {
|
||||
const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
|
||||
const uint i00 = idx - i01 * p.ne00;
|
||||
const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];
|
||||
if (expert < p.n_experts) {
|
||||
atomicAdd(vals[expert], 1);
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
|
||||
#ifdef USE_SUBGROUPS
|
||||
if (gl_SubgroupID == 0) {
|
||||
// pad the trip count so the subgroup ops stay in uniform control flow
|
||||
const uint n_experts_padded = (p.n_experts + gl_SubgroupSize - 1) & ~(gl_SubgroupSize - 1);
|
||||
uint base = 0;
|
||||
for (uint expert = gl_SubgroupInvocationID; expert < n_experts_padded; expert += gl_SubgroupSize) {
|
||||
const bool in_range = expert < p.n_experts;
|
||||
const uint count = in_range ? vals[expert] : 0;
|
||||
const uint offset = base + subgroupExclusiveAdd(count);
|
||||
if (in_range) {
|
||||
data_d[expert] = count;
|
||||
data_d[p.n_experts + expert] = offset;
|
||||
offsets[expert] = offset;
|
||||
cursors[expert] = 0;
|
||||
}
|
||||
base += subgroupAdd(count);
|
||||
}
|
||||
if (subgroupElect()) {
|
||||
data_d[2 * p.n_experts] = base;
|
||||
}
|
||||
}
|
||||
#else
|
||||
if (tid == 0) {
|
||||
uint offset = 0;
|
||||
for (uint expert = 0; expert < p.n_experts; ++expert) {
|
||||
const uint count = vals[expert];
|
||||
data_d[expert] = count;
|
||||
data_d[p.n_experts + expert] = offset;
|
||||
offsets[expert] = offset;
|
||||
cursors[expert] = 0;
|
||||
offset += count;
|
||||
}
|
||||
data_d[2 * p.n_experts] = offset;
|
||||
}
|
||||
#endif
|
||||
barrier();
|
||||
|
||||
for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {
|
||||
const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
|
||||
const uint i00 = idx - i01 * p.ne00;
|
||||
const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];
|
||||
if (expert < p.n_experts) {
|
||||
const uint row = atomicAdd(cursors[expert], 1);
|
||||
const uint packed_row_id = (i01 << 16) | (i00 & 0xffffu);
|
||||
data_d[2 * p.n_experts + 1 + offsets[expert] + row] = packed_row_id;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
uint count = 0;
|
||||
for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {
|
||||
const uint i01 = idx / p.ne00;
|
||||
const uint i00 = idx % p.ne00;
|
||||
const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
|
||||
const uint i00 = idx - i01 * p.ne00;
|
||||
const uint a = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];
|
||||
|
||||
count += uint(a == expert_id);
|
||||
|
||||
@@ -88,6 +88,9 @@ layout (push_constant) uniform parameter
|
||||
uint nei1;
|
||||
uint nbi1;
|
||||
uint ne11;
|
||||
uint padded_N;
|
||||
uint n_experts;
|
||||
uint hoist_row_ids;
|
||||
#else
|
||||
uint base_work_group_z;
|
||||
uint num_batches;
|
||||
@@ -214,27 +217,31 @@ void main() {
|
||||
const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK;
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
#ifdef MUL_MAT_ID_USE_SUBGROUPS
|
||||
if (bitCount(p.nei0) == 1) {
|
||||
load_row_ids(expert_idx, true, ic);
|
||||
if (p.hoist_row_ids != 0) {
|
||||
load_row_ids_hoisted(expert_idx, ic);
|
||||
} else {
|
||||
load_row_ids(expert_idx, false, ic);
|
||||
}
|
||||
#ifdef MUL_MAT_ID_USE_SUBGROUPS
|
||||
if (bitCount(p.nei0) == 1) {
|
||||
load_row_ids(expert_idx, true, ic);
|
||||
} else {
|
||||
load_row_ids(expert_idx, false, ic);
|
||||
}
|
||||
#else
|
||||
_ne1 = 0;
|
||||
for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
|
||||
for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
|
||||
if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
|
||||
if (_ne1 >= ic * BN) {
|
||||
row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
|
||||
_ne1 = 0;
|
||||
for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
|
||||
for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
|
||||
if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
|
||||
if (_ne1 >= ic * BN) {
|
||||
row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
|
||||
}
|
||||
_ne1++;
|
||||
}
|
||||
_ne1++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
barrier();
|
||||
barrier();
|
||||
#endif
|
||||
}
|
||||
|
||||
// Workgroup has no work
|
||||
if (ic * BN >= _ne1) return;
|
||||
|
||||
@@ -67,6 +67,10 @@ layout (push_constant) uniform parameter
|
||||
#endif
|
||||
// N dimension for the B matrix can be >= p.N
|
||||
uint padded_N;
|
||||
#ifdef MUL_MAT_ID
|
||||
uint n_experts;
|
||||
uint hoist_row_ids;
|
||||
#endif
|
||||
} p;
|
||||
|
||||
|
||||
@@ -225,6 +229,23 @@ void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {
|
||||
}
|
||||
barrier();
|
||||
}
|
||||
|
||||
void load_row_ids_hoisted(uint expert_idx, uint ic) {
|
||||
_ne1 = uint(data_expert_count[expert_idx]);
|
||||
|
||||
const uint tile_begin = ic * BN;
|
||||
const uint tile_count = tile_begin < _ne1 ? min(BN, _ne1 - tile_begin) : 0;
|
||||
const uint expert_offset = uint(data_expert_count[p.n_experts + expert_idx]);
|
||||
const uint row_ids_offset = 2 * p.n_experts + 1 + expert_offset + tile_begin;
|
||||
|
||||
for (uint i = gl_LocalInvocationIndex; i < tile_count; i += BLOCK_SIZE) {
|
||||
const uint packed_row_id = uint(data_expert_count[row_ids_offset + i]);
|
||||
const uint ii0 = packed_row_id & 0xffffu;
|
||||
const uint ii1 = packed_row_id >> 16;
|
||||
row_ids[i] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0);
|
||||
}
|
||||
barrier();
|
||||
}
|
||||
#endif
|
||||
|
||||
void main() {
|
||||
@@ -266,7 +287,9 @@ void main() {
|
||||
const uint ik = gl_WorkGroupID.x / blocks_m;
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
if (bitCount(p.nei0) == 1) {
|
||||
if (p.hoist_row_ids != 0) {
|
||||
load_row_ids_hoisted(expert_idx, ic);
|
||||
} else if (bitCount(p.nei0) == 1) {
|
||||
load_row_ids(expert_idx, true, ic);
|
||||
} else {
|
||||
load_row_ids(expert_idx, false, ic);
|
||||
|
||||
@@ -71,4 +71,19 @@ void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {
|
||||
barrier();
|
||||
}
|
||||
#endif // MUL_MAT_ID_USE_SUBGROUPS
|
||||
|
||||
void load_row_ids_hoisted(uint expert_idx, uint ic) {
|
||||
_ne1 = uint(data_expert_count[expert_idx]);
|
||||
|
||||
const uint tile_begin = ic * BN;
|
||||
const uint tile_count = tile_begin < _ne1 ? min(BN, _ne1 - tile_begin) : 0;
|
||||
const uint expert_offset = uint(data_expert_count[p.n_experts + expert_idx]);
|
||||
const uint row_ids_offset = 2 * p.n_experts + 1 + expert_offset + tile_begin;
|
||||
|
||||
for (uint i = gl_LocalInvocationIndex; i < tile_count; i += BLOCK_SIZE) {
|
||||
const uint packed_row_id = uint(data_expert_count[row_ids_offset + i]);
|
||||
row_ids[i] = u16vec2(packed_row_id & 0xffffu, packed_row_id >> 16);
|
||||
}
|
||||
barrier();
|
||||
}
|
||||
#endif // MUL_MAT_ID
|
||||
|
||||
@@ -56,6 +56,9 @@ layout (push_constant) uniform parameter
|
||||
uint nei1;
|
||||
uint nbi1;
|
||||
uint ne11;
|
||||
uint padded_N;
|
||||
uint n_experts;
|
||||
uint hoist_row_ids;
|
||||
#else
|
||||
uint base_work_group_z;
|
||||
uint num_batches;
|
||||
@@ -157,27 +160,31 @@ void main() {
|
||||
const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK;
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
#ifdef MUL_MAT_ID_USE_SUBGROUPS
|
||||
if (bitCount(p.nei0) == 1) {
|
||||
load_row_ids(expert_idx, true, ic);
|
||||
if (p.hoist_row_ids != 0) {
|
||||
load_row_ids_hoisted(expert_idx, ic);
|
||||
} else {
|
||||
load_row_ids(expert_idx, false, ic);
|
||||
}
|
||||
#ifdef MUL_MAT_ID_USE_SUBGROUPS
|
||||
if (bitCount(p.nei0) == 1) {
|
||||
load_row_ids(expert_idx, true, ic);
|
||||
} else {
|
||||
load_row_ids(expert_idx, false, ic);
|
||||
}
|
||||
#else
|
||||
_ne1 = 0;
|
||||
for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
|
||||
for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
|
||||
if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
|
||||
if (_ne1 >= ic * BN) {
|
||||
row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
|
||||
_ne1 = 0;
|
||||
for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
|
||||
for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
|
||||
if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
|
||||
if (_ne1 >= ic * BN) {
|
||||
row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
|
||||
}
|
||||
_ne1++;
|
||||
}
|
||||
_ne1++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
barrier();
|
||||
barrier();
|
||||
#endif
|
||||
}
|
||||
|
||||
// Workgroup has no work
|
||||
if (ic * BN >= _ne1) return;
|
||||
|
||||
@@ -1039,6 +1039,7 @@ void process_shaders() {
|
||||
string_to_spv("cumsum_multipass2_f32", "cumsum_multipass2.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
|
||||
|
||||
string_to_spv("count_experts", "count_experts.comp", merge_maps(base_dict, {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}}));
|
||||
string_to_spv("count_experts_subgroup", "count_experts.comp", merge_maps(base_dict, {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}, {"USE_SUBGROUPS", "1"}}));
|
||||
|
||||
for (std::string dim_str : {"", "_3d"}) {
|
||||
for (bool bda : {false, true}) {
|
||||
|
||||
@@ -467,10 +467,15 @@ class GGUFWriter:
|
||||
shard_bar.reset(total=(total if total > 0 else None))
|
||||
|
||||
# relying on the fact that Python dicts preserve insertion order (since 3.7)
|
||||
for ti in tensors.values():
|
||||
for name, ti in tensors.items():
|
||||
assert ti.tensor is not None # can only iterate once over the tensors
|
||||
assert ti.tensor.nbytes == ti.nbytes
|
||||
start = fout.tell()
|
||||
ti.tensor.tofile(fout)
|
||||
# a short write here would only surface as a corrupt file at load time
|
||||
if fout.tell() - start != ti.nbytes:
|
||||
raise ValueError(
|
||||
f"tensor {name!r} wrote {fout.tell() - start} bytes, expected {ti.nbytes}")
|
||||
if shard_bar is not None:
|
||||
shard_bar.update(ti.nbytes)
|
||||
if bar is not None:
|
||||
|
||||
@@ -251,6 +251,10 @@ class LazyChunkedTensor:
|
||||
def numpy(self) -> LazyChunkedTensor:
|
||||
return self
|
||||
|
||||
def __array__(self, *args, **kwargs):
|
||||
# numpy would otherwise make a 1-element object array of self, and write 8 bytes
|
||||
raise TypeError("LazyChunkedTensor cannot become an ndarray, it is written in chunks")
|
||||
|
||||
def quantize(self, qtype: Any) -> LazyChunkedTensor:
|
||||
from .constants import GGMLQuantizationType
|
||||
from .quants import QuantError, quant_shape_to_byte_shape
|
||||
|
||||
@@ -231,10 +231,10 @@ llama_context::llama_context(
|
||||
|
||||
cparams.fused_gdn_ar = true;
|
||||
cparams.fused_gdn_ch = true;
|
||||
cparams.auto_fgdn = true;
|
||||
cparams.auto_fgdn = false;
|
||||
|
||||
cparams.fused_lid = true;
|
||||
cparams.auto_flid = true;
|
||||
cparams.fused_lid = true;
|
||||
cparams.auto_flid = false;
|
||||
|
||||
cparams.fused_dsv4_hc_pre = true;
|
||||
cparams.fused_dsv4_hc_comb = true;
|
||||
|
||||
@@ -8785,6 +8785,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_conv_transpose_2d({3, 2, 3, 1}, {2, 2, 1, 3}, 1, kernel_type));
|
||||
test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2, kernel_type));
|
||||
test_cases.emplace_back(new test_conv_transpose_2d({129, 63, 35, 1}, {3, 3, 48, 35}, 1, kernel_type));
|
||||
test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 2}, {3, 3, 1, 9}, 2, kernel_type)); // for multiple batches
|
||||
}
|
||||
|
||||
test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1}));
|
||||
|
||||
Reference in New Issue
Block a user