ClaudeBot/llama.cpp
0
0
Fork 1
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-17 20:31:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

openvino : Update OpenVINO to 2026.4;fix clangd,MSVC warnings; (#29009)

Browse Source 
* Update to openvino-2026.4

* Update OV docs

* ggml-openvino : fix clangd and MSVC warnings

* fix int to ptr cast, more internal linkage enforcement, and avoiding duplicate switch case

---------

Co-authored-by: Mostafa Faheem <mostafaaafaheem@gmail.com>
This commit is contained in:
Ravi Panchumarthy
2026-09-17 12:46:14 +02:00
committed by GitHub
co-authored by Mostafa Faheem
parent 4ff829ec2e
commit ebbb185227
30 changed files with 948 additions and 1105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.devops/openvino.Dockerfile
+6 -6
View File
@@ -1,5 +1,5 @@
ARG OPENVINO_VERSION_MAJOR=2026.3.1
ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d
ARG OPENVINO_VERSION_MAJOR=2026.4
ARG OPENVINO_VERSION_FULL=2026.4.0.22959.99c81491cc3
ARG UBUNTU_VERSION=24.04
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
@@ -10,9 +10,9 @@ ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
ARG IGDGMM_VERSION=22.10.0
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
ARG NPU_DRIVER_VERSION=v1.35.0
ARG NPU_DRIVER_FULL=v1.35.0.20260722-29947505341
ARG LIBZE1_VERSION=1.28.2-1~24.04~ppa1
ARG NPU_DRIVER_VERSION=v1.38.0
ARG NPU_DRIVER_FULL=v1.38.0.20260910-34487311128
ARG LIBZE1_VERSION=1.32.0-1~24.04~ppa1
# Optional proxy build arguments
ARG http_proxy=
@@ -173,7 +173,7 @@ RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \
fi; \
DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \
if [ ! -f "$DEB" ]; then \
wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260606T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260830T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
fi; \
mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \
apt-get update; \
.github/workflows/build-cache.yml
+4 -4
View File
@@ -41,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
OPENVINO_VERSION_MAJOR: "2026.4"
OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
@@ -69,8 +69,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
OPENVINO_VERSION_MAJOR: "2026.4"
OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
.github/workflows/build-openvino.yml
+4 -4
View File
@@ -41,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
OPENVINO_VERSION_MAJOR: "2026.4"
OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
@@ -96,8 +96,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
OPENVINO_VERSION_MAJOR: "2026.4"
OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
.github/workflows/build-self-hosted.yml
+2 -2
View File
@@ -412,8 +412,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
OPENVINO_VERSION_MAJOR: "2026.4"
OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
.github/workflows/release.yml
+4 -4
View File
@@ -555,8 +555,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
OPENVINO_VERSION_MAJOR: "2026.4"
OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Set OpenVINO version output
@@ -669,8 +669,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.3.1"
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
OPENVINO_VERSION_MAJOR: "2026.4"
OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Set OpenVINO version output
docs/backend/OPENVINO.md
+14 -12
View File
@@ -12,6 +12,8 @@ The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a t
- Compiles and caches the model for the target device.
- Binds GGML tensor memory to OpenVINO inference tensors and runs inference.
For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend Contributing Guide](https://github.com/ravi9/llamacpp-ov-dev-guide/blob/main/contributing-llamacpp-ov.md).
## Contents
- [Supported Devices](#supported-devices)
@@ -96,7 +98,7 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
- Note: The NPU operates in stateless mode only.
- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.35.0.
- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel Graphics Compiler 2.41.5 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.38.0.
- See [Known Limitations](#known-limitations) for context on observed failures.
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
@@ -117,9 +119,9 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
| | | | |
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
| | | | |
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
@@ -132,9 +134,9 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ✗ / ✗ | ✗ |
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
@@ -242,8 +244,8 @@ chmod +x build-llamacpp-ov.sh
# ============================================
set -euo pipefail
OPENVINO_VERSION_MAJOR="2026.3.1"
OPENVINO_VERSION_FULL="2026.3.1.22476.56d9685302d"
OPENVINO_VERSION_MAJOR="2026.4"
OPENVINO_VERSION_FULL="2026.4.0.22959.99c81491cc3"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}"
@@ -340,7 +342,7 @@ echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf"
```
> [!NOTE]
> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
> The script pins OpenVINO `2026.4` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
</details>
@@ -370,8 +372,8 @@ REM ============================================
REM llama.cpp OpenVINO Build Script (Ninja)
REM ============================================
set "OPENVINO_VERSION_MAJOR=2026.3.1"
set "OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d"
set "OPENVINO_VERSION_MAJOR=2026.4"
set "OPENVINO_VERSION_FULL=2026.4.0.22959.99c81491cc3"
set "SCRIPT_DIR=%~dp0"
set "VCPKG_DIR=C:\vcpkg"
@@ -550,7 +552,7 @@ endlocal
```
> [!NOTE]
> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
> The script pins OpenVINO `2026.4` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
</details>
ggml/src/ggml-openvino/ggml-decoder.cpp
+10 -13
View File
@@ -245,7 +245,7 @@ void GgmlOvDecoder::set_input_output() {
if (src->op == GGML_OP_VIEW) {
// Traverse upward through nested VIEW operations
std::remove_reference_t<decltype(current_node_info.node_inputs_views[src_name])> view_chain;
auto current = src;
auto * current = src;
while (current != nullptr) {
auto current_name = get_tensor_ov_name(m_cgraph, current);
@@ -612,9 +612,8 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
if (node->src[1]->view_src != nullptr) {
if (node->src[3] != nullptr) {
return 4; // decoder self-attention
} else {
return 5; // cross-attention or encoder self-attention
};
}
return 5; // cross-attention or encoder self-attention
}
break;
default:
@@ -736,8 +735,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
bool rope_seen = false;
for (int i = 0; i < cgraph->n_nodes; i++) {
auto * node = cgraph->nodes[i];
std::string name = std::string(node->name);
ggml_tensor * node = cgraph->nodes[i];
const int attention_pattern_case = get_attention_pattern_case(node);
if (attention_pattern_case != -1) {
ggml_tensor * cache_k_permute = nullptr;
@@ -948,7 +946,6 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
if (m_naive) {
return input != nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)};
}
auto name = std::string(input->name);
ov::PartialShape input_shape;
if (is_inp_tok(input, op) || is_inp_pos(input, op)) {
@@ -1474,7 +1471,7 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
void GgmlOvDecoder::dump_cgraph(const ggml_cgraph * cgraph, std::string & filename) {
std::ofstream file(filename);
if (!file.is_open()) {
std::cerr << "Failed to open file" << std::endl;
std::cerr << "Failed to open file" << '\n';
return;
}
@@ -1580,11 +1577,11 @@ void print_tensor_address_map(const ggml_cgraph * cgraph) {
}
}
for (const auto & pair : address_map) {
std::cout << "Address: " << pair.first << std::endl;
std::cout << "Address: " << pair.first << '\n';
for (const auto & name : pair.second) {
std::cout << name << " ; ";
}
std::cout << std::endl << std::endl;
std::cout << "\n\n";
}
}
@@ -2226,7 +2223,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
std::cout << ", ";
}
}
std::cout << "]" << std::endl;
std::cout << "]" << '\n';
// print the src name & shape with the dynamic dim for debugging
for (int j = 0; j < GGML_MAX_SRC; j++) {
ggml_tensor * src = node->src[j];
@@ -2245,9 +2242,9 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
std::cout << ", ";
}
}
std::cout << "]" << std::endl;
std::cout << "]" << '\n';
}
std::cout << std::endl;
std::cout << '\n';
}
}
}
ggml/src/ggml-openvino/ggml-decoder.h
+12 -14
View File
@@ -354,41 +354,41 @@ public:
void update_io(ggml_cgraph * cgraph);
inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE;
}
inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_ROPE && tensor == op->src[1];
}
// IMROPE packs 4 stacked position planes (t/h/w/e) into inp_pos, each of length
// n_tokens; other modes carry a single position per token.
inline static int get_inp_pos_n_planes(const ggml_tensor * op) {
static int get_inp_pos_n_planes(const ggml_tensor * op) {
return op->op_params[2] == GGML_ROPE_TYPE_IMROPE ? 4 : 1;
}
inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) {
return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM;
}
inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) ||
(op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]);
}
inline static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_MUL_MAT && tensor == op->src[1] && tensor->op == GGML_OP_NONE &&
(tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->type == GGML_TYPE_F32 &&
op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE;
}
inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_ROPE && tensor == op->src[2];
}
// also returns true for cache_s and cache_r in SSM/DeltaNet models
inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
if (tensor == nullptr) {
return false;
}
@@ -396,14 +396,14 @@ public:
(op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor);
}
inline static bool is_conv_state_writeback(const ggml_tensor * node) {
static bool is_conv_state_writeback(const ggml_tensor * node) {
return node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) &&
node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr &&
node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src;
}
inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor;
}
@@ -411,13 +411,13 @@ public:
return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask;
}
inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE &&
op->src[1]->op == GGML_OP_NONE;
}
// the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp)
inline static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) {
static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] &&
op->src[0]->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY;
}
@@ -481,5 +481,3 @@ private:
};
void print_tensor_address_map(const ggml_cgraph * cgraph);
std::optional<int> extract_layer_from_name(const std::string & name);
ggml/src/ggml-openvino/ggml-openvino-extra.cpp
-4
View File
@@ -472,10 +472,6 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
switch (tensor->type) {
case GGML_TYPE_MXFP4:
layout.is_u4 = true;
layout.is_symmetric = true;
break;
case GGML_TYPE_Q4_0:
layout.is_u4 = true;
layout.is_symmetric = true;
ggml/src/ggml-openvino/ggml-openvino.cpp
+17 -14
View File
@@ -28,12 +28,7 @@
#include <string>
#include <vector>
#ifndef _WIN32
# include <sys/mman.h>
# include <unistd.h>
#endif
#if defined(_WIN32)
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
# define NOMINMAX
@@ -61,6 +56,7 @@
// - CPU repack buffer: tensor->extra stores tensor_traits with repacked data
// =====================================================
namespace {
// Buffer context that manages per-tensor allocations (no contiguous buffer for weights)
struct ggml_backend_openvino_buffer_context {
int device;
@@ -199,6 +195,7 @@ struct ggml_backend_openvino_buffer_type_context {
int device;
std::string name;
};
} // namespace
// =====================================================
// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS)
@@ -258,14 +255,16 @@ void ggml_openvino_release_weight_buffers() {
for (const auto & b : reg.buffers) {
// Align down/up to page boundaries so madvise only drops whole pages
// fully owned by this buffer.
const long page = sysconf(_SC_PAGESIZE);
uintptr_t start = reinterpret_cast<uintptr_t>(b.first);
uintptr_t end = start + b.second;
uintptr_t astart = (start + page - 1) & ~(uintptr_t) (page - 1);
uintptr_t aend = end & ~(uintptr_t) (page - 1);
if (aend > astart) {
if (madvise(reinterpret_cast<void *>(astart), aend - astart, MADV_DONTNEED) == 0) {
total += aend - astart;
const size_t page = (size_t) sysconf(_SC_PAGESIZE);
const uintptr_t ustart = reinterpret_cast<uintptr_t>(b.first);
const size_t offset_to_page = (page - (ustart & (page - 1))) & (page - 1);
if (b.second > offset_to_page) {
const size_t aligned_len = (b.second - offset_to_page) & ~(page - 1);
if (aligned_len > 0) {
char * astart = static_cast<char *>(b.first) + offset_to_page;
if (madvise(astart, aligned_len, MADV_DONTNEED) == 0) {
total += aligned_len;
}
}
}
}
@@ -876,11 +875,13 @@ GGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend) {
return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_openvino_guid());
}
namespace {
struct ggml_backend_openvino_device_context {
int device;
std::string name;
std::string description;
};
}
static const char * ggml_backend_openvino_device_get_name(ggml_backend_dev_t dev) {
ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;
@@ -1588,9 +1589,11 @@ static const struct ggml_backend_device_i ggml_backend_openvino_device_interface
/* .event_synchronize = */ NULL,
};
namespace {
struct ggml_backend_openvino_reg_context {
std::vector<ggml_backend_dev_t> devices;
};
}
static const char * ggml_backend_openvino_reg_get_name(ggml_backend_reg_t reg) {
return GGML_OPENVINO_NAME;
ggml/src/ggml-openvino/ggml-quants.cpp
+396 -364
View File
@@ -34,6 +34,15 @@
#include <string>
#include <vector>
// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp
namespace ov::op::util {
// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp
bool get_single_value(const std::shared_ptr<ov::op::v0::Constant> & const_node,
float & value,
bool check_value_range = true);
} // namespace ov::op::util
namespace {
void unpack_32_4(const uint8_t * data, uint8_t * dst) {
std::fill_n(dst, 16, 0);
for (int j = 0; j < 16; ++j) {
@@ -48,11 +57,11 @@ void unpack_32_4(const uint8_t * data, uint8_t * dst) {
}
}
static constexpr size_t MXFP4_BLOCK_SIZE = 32;
static constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2;
static constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE;
constexpr size_t MXFP4_BLOCK_SIZE = 32;
constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2;
constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE;
static void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) {
void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) {
for (int j = 0; j < static_cast<int>(MXFP4_BLOCK_QS_SIZE); j += 2) {
const uint8_t v0 = data[j] & 0x0F;
const uint8_t v1 = (data[j + 1] & 0x0F) << 4;
@@ -419,7 +428,7 @@ void extract_q6_k_data(const ggml_tensor * tensor,
}
}
static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {
inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {
if (j < 4) {
*d = q[j] & 63;
*m = q[j + 4] & 63;
@@ -514,9 +523,9 @@ void extract_q5_k_data(const ggml_tensor * tensor,
ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
ov::Tensor & scales,
ov::Tensor & zp,
size_t group_size,
bool use_bias,
bool for_gather_matmul) {
size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
bool use_bias = false,
bool for_gather_matmul = false) {
ov::Shape orig_shape = weight.get_shape();
bool is_signed = (weight.get_element_type() == ov::element::i8); // Symmetric: signed weights, no ZP
@@ -611,13 +620,24 @@ ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
return std::make_shared<ov::op::v0::Convert>(result, ov::element::f32);
}
// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights
// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as
// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in
// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see
// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul
// directly from the resulting f16 dequant chain.
//
// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp
// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point);
// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not
// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern.
// See make_int8_weights for the meaning of for_gather_matmul.
ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
ov::Tensor & scales,
ov::Tensor & zp,
size_t group_size,
bool use_bias,
bool for_gather_matmul) {
size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
bool use_bias = false,
bool for_gather_matmul = false) {
ov::Shape orig_weight_shape = weight.get_shape();
bool is_signed = (weight.get_element_type() == ov::element::i4); // Symmetric: signed weights, no ZP
@@ -746,357 +766,6 @@ ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight) {
return weights_node;
}
// Extract quantized weights from tensor and create weight subgraph
std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
const void * data,
ov::Tensor & weights,
ov::Tensor & scales,
ov::Tensor & zp,
bool use_bias) {
// Create a temporary tensor for extraction functions that read from tensor->data
ggml_tensor temp_tensor = *tensor;
temp_tensor.data = const_cast<void *>(data);
if (tensor->type == GGML_TYPE_MXFP4) {
extract_mxfp4_data(&temp_tensor, weights, scales);
auto result = make_mxfp4_weights(weights, scales).get_node_shared_ptr();
result->set_friendly_name(tensor->name);
return result;
}
// Determine block size based on tensor type
int64_t weights_per_block;
bool is_u4;
switch (tensor->type) {
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q4_K:
is_u4 = true;
weights_per_block = 32;
break;
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q5_K:
is_u4 = false;
weights_per_block = 32;
break;
case GGML_TYPE_Q6_K:
is_u4 = false;
weights_per_block = 16;
break;
default:
throw std::runtime_error("Unsupported quantized type for extraction: " +
std::string(ggml_type_name(tensor->type)));
}
// 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction
// (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point --
// round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses
// into GatherMatmulCompressed since it stays a Subtract, not an Add.
const bool for_gather_matmul = tensor->ne[2] > 1;
use_bias = use_bias || for_gather_matmul;
// Extract quantized data
switch (tensor->type) {
case GGML_TYPE_Q4_0:
extract_q4_0_data(&temp_tensor, weights, scales, zp);
break;
case GGML_TYPE_Q4_1:
extract_q4_1_data(&temp_tensor, weights, scales, zp, use_bias);
break;
case GGML_TYPE_Q4_K:
extract_q4_k_data(&temp_tensor, weights, scales, zp, use_bias);
break;
case GGML_TYPE_Q5_1:
extract_q5_1_data(&temp_tensor, weights, scales, zp, use_bias);
break;
case GGML_TYPE_Q8_0:
extract_q8_0_data(&temp_tensor, weights, scales, zp);
break;
case GGML_TYPE_Q6_K:
extract_q6_k_data(&temp_tensor, weights, scales, zp);
break;
case GGML_TYPE_Q5_K:
extract_q5_k_data(&temp_tensor, weights, scales, zp, use_bias);
break;
default:
throw std::runtime_error("Unsupported quantized type: " + std::string(ggml_type_name(tensor->type)));
}
// Create the OpenVINO weight subgraph. 3D expert weights (MoE) are routed through the
// GatherMatmul-oriented path: dequantized in f16, with constant folding disabled on the chain.
ov::Output<ov::Node> weight_node;
if (is_u4) {
weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul);
} else {
weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul);
}
auto result = weight_node.get_node_shared_ptr();
result->set_friendly_name(tensor->name);
return result;
}
// Requantize weights to target format, writing to provided buffers
std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
const void * data,
ExtraQuantType requant_type,
int64_t block_size,
ov::Tensor & weights,
ov::Tensor & scales,
ov::Tensor & zp) {
int64_t n_elements = ggml_nelements(tensor);
const int64_t ne0 = tensor->ne[0]; // elements per row
const int64_t n_rows = n_elements / ne0;
const auto * type_traits = ggml_get_type_traits(tensor->type);
const size_t src_row_bytes = ggml_row_size(tensor->type, ne0);
bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128 ||
requant_type == ExtraQuantType::Q4_0_64 || requant_type == ExtraQuantType::Q4_1_64);
// Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or
// GGML_OPENVINO_MEMORY_OPTIMIZE): instead of
// materializing the full n_elements F32 array (e.g. ~1 GB for token_embd), dequantize
// a chunk of complete rows into a small scratch and quantize/convert it straight into
// the output buffers, capping the transient F32 footprint at CHUNK_ROWS*ne0 floats.
//
// Only valid (and only used) for the Q8_0_C / Q8_1_C / F16 targets whose block size
// divides a row (channel-wise _C uses block_size == ne0) so no target block straddles
// a row boundary, and Q8/F16 have no cross-block packing. The u4 (Q4_0) path packs two
// weights per byte with running zp ORs that assume a single whole-array call, so it is
// never streamed. When the flag is off, behavior is identical to the original
// full-materialization path.
const bool stream_requant = ggml_openvino_reduce_compile_mem_enabled() && !is_u4 &&
!(block_size > 0 && ne0 % block_size != 0);
if (!stream_requant) {
// Full materialization (original behavior): dequantize the whole tensor to F32,
// then convert/quantize in one call.
std::vector<float> weights_f32(n_elements);
type_traits->to_float(data, weights_f32.data(), n_elements);
if (requant_type == ExtraQuantType::F16) {
ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements);
auto result = std::make_shared<ov::op::v0::Constant>(weights);
result->set_friendly_name(tensor->name);
return result;
}
if (requant_type == ExtraQuantType::Q4_1_64) {
quantize_q4_1_asym(weights_f32.data(), weights, scales, zp, n_elements, block_size);
} else if (is_u4) {
quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
} else if (requant_type == ExtraQuantType::Q8_1_C) {
quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size);
} else {
quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
}
} else {
// Streaming path for Q8_0_C / Q8_1_C / F16 (covers token_embd, output.weight,
// and per-layer Q6_K/Q5_K requant — the large transient cases).
const int64_t CHUNK_ROWS = std::min<int64_t>(n_rows, 256);
std::vector<float> scratch(CHUNK_ROWS * ne0);
// F16 destination: 2 bytes/element, advanced per chunk by r0*ne0 elements.
auto * f16_base = static_cast<uint8_t *>(weights.data());
for (int64_t r0 = 0; r0 < n_rows; r0 += CHUNK_ROWS) {
const int64_t rows = std::min(CHUNK_ROWS, n_rows - r0);
const int64_t elems = rows * ne0;
const auto * src = static_cast<const uint8_t *>(data) + r0 * src_row_bytes;
type_traits->to_float(src, scratch.data(), elems);
if (requant_type == ExtraQuantType::F16) {
ggml_get_type_traits(GGML_TYPE_F16)
->from_float_ref(scratch.data(), f16_base + (r0 * ne0) * sizeof(uint16_t), elems);
} else {
const int64_t block_offset = (r0 * ne0) / block_size;
if (requant_type == ExtraQuantType::Q8_1_C) {
quantize_q8_1(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
} else {
quantize_q8_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
}
}
}
if (requant_type == ExtraQuantType::F16) {
auto result = std::make_shared<ov::op::v0::Constant>(weights);
result->set_friendly_name(tensor->name);
return result;
}
}
// Create the OpenVINO weight subgraph
ov::Output<ov::Node> weight_node;
if (is_u4) {
weight_node = make_int4_weights(weights, scales, zp, block_size);
} else {
weight_node = make_int8_weights(weights, scales, zp, block_size);
}
auto result = weight_node.get_node_shared_ptr();
result->set_friendly_name(tensor->name);
return result;
}
OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) {
GGML_ASSERT(tensor != nullptr);
GGML_ASSERT(data != nullptr);
OvWeight result;
// Get shape for weights: [rows, cols], or [n_expert, rows, cols] for 3D MoE expert weights.
ov::Shape node_shape = (tensor->ne[2] > 1) ?
ov::Shape{static_cast<size_t>(tensor->ne[2]), static_cast<size_t>(tensor->ne[1]),
static_cast<size_t>(tensor->ne[0])} :
ov::Shape{static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
// Handle F16/F32/BF16 weights
if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {
ov::element::Type element_type;
switch (tensor->type) {
case GGML_TYPE_F32:
element_type = ov::element::f32;
break;
case GGML_TYPE_F16:
element_type = ov::element::f16;
break;
case GGML_TYPE_BF16:
element_type = ov::element::bf16;
break;
default:
OPENVINO_THROW("Unexpected tensor type in F16/F32/BF16 path");
}
if (output_base_ptr && output_base_ptr != data) {
// Using external buffer - copy data and create shared-memory constant
size_t tensor_bytes = ggml_nbytes(tensor);
memcpy(output_base_ptr, data, tensor_bytes);
result.weights = ov::Tensor(element_type, node_shape, output_base_ptr);
} else {
result.weights = ov::Tensor(element_type, node_shape, data);
}
result.weight_node = std::make_shared<ov::op::v0::Constant>(result.weights);
return result;
}
// Handle quantized weights
if (!ggml_is_quantized(tensor->type)) {
OPENVINO_THROW("Unsupported weight tensor type: ", ggml_type_name(tensor->type));
}
result.layout = ggml_openvino_get_extracted_layout(tensor, use_bias);
const auto & layout = result.layout;
if (layout.total_size == 0) {
OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type));
}
// 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see
// extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul"
// check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly.
// Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose
// zp sizing/type is unaffected by for_gather_matmul, so they are excluded here.
const bool for_gather_matmul = tensor->ne[2] > 1;
const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul);
const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1);
if (is_3d_mxfp4_moe) {
ov::Shape packed_shape = {static_cast<size_t>(tensor->ne[3]),
static_cast<size_t>(tensor->ne[2]),
static_cast<size_t>(tensor->ne[1]),
static_cast<size_t>(tensor->ne[0] / MXFP4_BLOCK_SIZE),
MXFP4_BLOCK_BYTES};
const size_t tensor_bytes = ggml_nbytes(tensor);
if (output_base_ptr) {
auto * buf_base = static_cast<uint8_t *>(output_base_ptr);
memcpy(buf_base + layout.weights_offset, data, tensor_bytes);
result.weights = ov::Tensor(ov::element::u8, packed_shape, buf_base + layout.weights_offset);
} else {
result.weights = ov::Tensor(ov::element::u8, packed_shape);
memcpy(result.weights.data(), data, tensor_bytes);
}
result.weight_node = make_mxfp4_moe_packed_weights(result.weights).get_node_shared_ptr();
result.weight_node->set_friendly_name(tensor->name);
return result;
}
if (use_bias) {
OPENVINO_ASSERT(!layout.is_requant,
"use_bias is only used for test-backend-ops, which should not have requantization");
// bias node will be created on the fly and not use backend buffer
output_base_ptr = nullptr;
}
// F16 requant path - no separate scales/zp needed in result
if (layout.is_requant && layout.requant_type.has_value() && layout.requant_type.value() == ExtraQuantType::F16) {
if (output_base_ptr) {
result.weights = ov::Tensor(ov::element::f16, node_shape,
static_cast<uint8_t *>(output_base_ptr) + layout.weights_offset);
} else {
result.weights = ov::Tensor(ov::element::f16, node_shape);
}
ov::Tensor dummy_scales, dummy_zp; // Not used for F16
result.weight_node =
requantize_to_buffers(tensor, data, ExtraQuantType::F16, 0, result.weights, dummy_scales, dummy_zp);
return result;
}
// Quantized path (normal extraction or quantized requant)
// Create weight/scale/zp tensors - shared between both paths
// For symmetric quantization, use signed types (i4/i8) and no ZP tensor
ov::element::Type weight_type = tensor->type == GGML_TYPE_MXFP4 ?
ov::element::f4e2m1 :
(layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) :
(layout.is_u4 ? ov::element::u4 : ov::element::u8));
ov::Shape scale_shape = node_shape;
scale_shape.back() /= layout.weights_per_block;
if (tensor->type == GGML_TYPE_MXFP4) {
if (tensor->ne[2] == 1 && tensor->ne[3] == 1) {
node_shape = {static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
} else {
node_shape.clear();
for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {
node_shape.push_back(static_cast<size_t>(tensor->ne[i]));
}
}
scale_shape = node_shape;
scale_shape.back() /= layout.weights_per_block;
}
if (output_base_ptr) {
uint8_t * buf_base = static_cast<uint8_t *>(output_base_ptr);
result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset);
const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset);
if (!layout.is_symmetric) {
ov::element::Type zp_type =
zp_is_f16 ? ov::element::f16 : (layout.is_u4 ? ov::element::u4 : ov::element::u8);
result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset);
}
// else: result.zp remains default-constructed (empty) for symmetric
} else {
result.weights = ov::Tensor(weight_type, node_shape);
const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
result.scales = ov::Tensor(scale_type, scale_shape);
if (!layout.is_symmetric) {
if (zp_is_f16) {
result.zp = ov::Tensor(ov::element::f16, scale_shape);
} else {
ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
result.zp = ov::Tensor(zp_type, scale_shape);
}
}
// else: result.zp remains default-constructed (empty) for symmetric
}
if (layout.is_requant && layout.requant_type.has_value()) {
result.weight_node = requantize_to_buffers(tensor, data, layout.requant_type.value(), layout.weights_per_block,
result.weights, result.scales, result.zp);
} else {
result.weight_node =
extract_quantized_weights(tensor, data, result.weights, result.scales, result.zp, use_bias);
}
return result;
}
void quantize_q4_0(const float * x,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
@@ -1252,7 +921,7 @@ void quantize_q8_0(const float * x,
ov::Tensor & zp_arr,
int64_t k,
int64_t qk,
int64_t block_offset) {
int64_t block_offset = 0) {
assert(k % qk == 0);
const int nb = k / qk;
@@ -1308,7 +977,7 @@ void quantize_q8_1(const float * x,
ov::Tensor & zp_arr,
int64_t k,
int64_t qk,
int64_t block_offset) {
int64_t block_offset = 0) {
assert(k % qk == 0);
const int nb = k / qk;
@@ -1339,3 +1008,366 @@ void quantize_q8_1(const float * x,
}
}
}
// Extract quantized weights from tensor and create weight subgraph
// If weights/scales/zp are provided (non-empty), uses them as output buffers
// Otherwise allocates new ov::Tensors internally
// Returns the weight node (make_int4_weights or make_int8_weights result)
std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
const void * data, // Source data pointer (may differ from tensor->data)
ov::Tensor & weights,
ov::Tensor & scales,
ov::Tensor & zp,
// Use an exact f16 zero point (vs. a rounded integer one); always
// used for for_gather_matmul (3D MoE expert) weights regardless of
// this flag, and also settable explicitly for test-backend-ops.
bool use_bias = false) {
// Create a temporary tensor for extraction functions that read from tensor->data
ggml_tensor temp_tensor = *tensor;
temp_tensor.data = const_cast<void *>(data);
if (tensor->type == GGML_TYPE_MXFP4) {
extract_mxfp4_data(&temp_tensor, weights, scales);
auto result = make_mxfp4_weights(weights, scales).get_node_shared_ptr();
result->set_friendly_name(tensor->name);
return result;
}
// Determine block size based on tensor type
int64_t weights_per_block;
bool is_u4;
switch (tensor->type) {
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q4_K:
is_u4 = true;
weights_per_block = 32;
break;
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q5_K:
is_u4 = false;
weights_per_block = 32;
break;
case GGML_TYPE_Q6_K:
is_u4 = false;
weights_per_block = 16;
break;
default:
throw std::runtime_error("Unsupported quantized type for extraction: " +
std::string(ggml_type_name(tensor->type)));
}
// 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction
// (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point --
// round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses
// into GatherMatmulCompressed since it stays a Subtract, not an Add.
const bool for_gather_matmul = tensor->ne[2] > 1;
use_bias = use_bias || for_gather_matmul;
// Extract quantized data
switch (tensor->type) {
case GGML_TYPE_Q4_0:
extract_q4_0_data(&temp_tensor, weights, scales, zp);
break;
case GGML_TYPE_Q4_1:
extract_q4_1_data(&temp_tensor, weights, scales, zp, use_bias);
break;
case GGML_TYPE_Q4_K:
extract_q4_k_data(&temp_tensor, weights, scales, zp, use_bias);
break;
case GGML_TYPE_Q5_1:
extract_q5_1_data(&temp_tensor, weights, scales, zp, use_bias);
break;
case GGML_TYPE_Q8_0:
extract_q8_0_data(&temp_tensor, weights, scales, zp);
break;
case GGML_TYPE_Q6_K:
extract_q6_k_data(&temp_tensor, weights, scales, zp);
break;
case GGML_TYPE_Q5_K:
extract_q5_k_data(&temp_tensor, weights, scales, zp, use_bias);
break;
default:
throw std::runtime_error("Unsupported quantized type: " + std::string(ggml_type_name(tensor->type)));
}
// Create the OpenVINO weight subgraph. 3D expert weights (MoE) are routed through the
// GatherMatmul-oriented path: dequantized in f16, with constant folding disabled on the chain.
ov::Output<ov::Node> weight_node;
if (is_u4) {
weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul);
} else {
weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul);
}
auto result = weight_node.get_node_shared_ptr();
result->set_friendly_name(tensor->name);
return result;
}
// Requantize weights from tensor to target format, writing to provided buffers
// For F16 target, only weights buffer is used (scales/zp ignored)
// Returns the weight node
std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
const void * data, // Source data pointer
ExtraQuantType requant_type,
int64_t block_size,
ov::Tensor & weights,
ov::Tensor & scales,
ov::Tensor & zp) {
int64_t n_elements = ggml_nelements(tensor);
const int64_t ne0 = tensor->ne[0]; // elements per row
const int64_t n_rows = n_elements / ne0;
const auto * type_traits = ggml_get_type_traits(tensor->type);
const size_t src_row_bytes = ggml_row_size(tensor->type, ne0);
bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128 ||
requant_type == ExtraQuantType::Q4_0_64 || requant_type == ExtraQuantType::Q4_1_64);
// Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or
// GGML_OPENVINO_MEMORY_OPTIMIZE): instead of
// materializing the full n_elements F32 array (e.g. ~1 GB for token_embd), dequantize
// a chunk of complete rows into a small scratch and quantize/convert it straight into
// the output buffers, capping the transient F32 footprint at CHUNK_ROWS*ne0 floats.
//
// Only valid (and only used) for the Q8_0_C / Q8_1_C / F16 targets whose block size
// divides a row (channel-wise _C uses block_size == ne0) so no target block straddles
// a row boundary, and Q8/F16 have no cross-block packing. The u4 (Q4_0) path packs two
// weights per byte with running zp ORs that assume a single whole-array call, so it is
// never streamed. When the flag is off, behavior is identical to the original
// full-materialization path.
const bool stream_requant = ggml_openvino_reduce_compile_mem_enabled() && !is_u4 &&
!(block_size > 0 && ne0 % block_size != 0);
if (!stream_requant) {
// Full materialization (original behavior): dequantize the whole tensor to F32,
// then convert/quantize in one call.
std::vector<float> weights_f32(n_elements);
type_traits->to_float(data, weights_f32.data(), n_elements);
if (requant_type == ExtraQuantType::F16) {
ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements);
auto result = std::make_shared<ov::op::v0::Constant>(weights);
result->set_friendly_name(tensor->name);
return result;
}
if (requant_type == ExtraQuantType::Q4_1_64) {
quantize_q4_1_asym(weights_f32.data(), weights, scales, zp, n_elements, block_size);
} else if (is_u4) {
quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
} else if (requant_type == ExtraQuantType::Q8_1_C) {
quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size);
} else {
quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
}
} else {
// Streaming path for Q8_0_C / Q8_1_C / F16 (covers token_embd, output.weight,
// and per-layer Q6_K/Q5_K requant — the large transient cases).
const int64_t CHUNK_ROWS = std::min<int64_t>(n_rows, 256);
std::vector<float> scratch(CHUNK_ROWS * ne0);
// F16 destination: 2 bytes/element, advanced per chunk by r0*ne0 elements.
auto * f16_base = static_cast<uint8_t *>(weights.data());
for (int64_t r0 = 0; r0 < n_rows; r0 += CHUNK_ROWS) {
const int64_t rows = std::min(CHUNK_ROWS, n_rows - r0);
const int64_t elems = rows * ne0;
const auto * src = static_cast<const uint8_t *>(data) + r0 * src_row_bytes;
type_traits->to_float(src, scratch.data(), elems);
if (requant_type == ExtraQuantType::F16) {
ggml_get_type_traits(GGML_TYPE_F16)
->from_float_ref(scratch.data(), f16_base + (r0 * ne0) * sizeof(uint16_t), elems);
} else {
const int64_t block_offset = (r0 * ne0) / block_size;
if (requant_type == ExtraQuantType::Q8_1_C) {
quantize_q8_1(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
} else {
quantize_q8_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
}
}
}
if (requant_type == ExtraQuantType::F16) {
auto result = std::make_shared<ov::op::v0::Constant>(weights);
result->set_friendly_name(tensor->name);
return result;
}
}
// Create the OpenVINO weight subgraph
ov::Output<ov::Node> weight_node;
if (is_u4) {
weight_node = make_int4_weights(weights, scales, zp, block_size);
} else {
weight_node = make_int8_weights(weights, scales, zp, block_size);
}
auto result = weight_node.get_node_shared_ptr();
result->set_friendly_name(tensor->name);
return result;
}
} // namespace
OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) {
GGML_ASSERT(tensor != nullptr);
GGML_ASSERT(data != nullptr);
OvWeight result;
// Get shape for weights: [rows, cols], or [n_expert, rows, cols] for 3D MoE expert weights.
ov::Shape node_shape = (tensor->ne[2] > 1) ?
ov::Shape{static_cast<size_t>(tensor->ne[2]), static_cast<size_t>(tensor->ne[1]),
static_cast<size_t>(tensor->ne[0])} :
ov::Shape{static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
// Handle F16/F32/BF16 weights
if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {
ov::element::Type element_type;
switch (tensor->type) {
case GGML_TYPE_F32:
element_type = ov::element::f32;
break;
case GGML_TYPE_F16:
element_type = ov::element::f16;
break;
case GGML_TYPE_BF16:
element_type = ov::element::bf16;
break;
default:
OPENVINO_THROW("Unexpected tensor type in F16/F32/BF16 path");
}
if (output_base_ptr && output_base_ptr != data) {
// Using external buffer - copy data and create shared-memory constant
size_t tensor_bytes = ggml_nbytes(tensor);
memcpy(output_base_ptr, data, tensor_bytes);
result.weights = ov::Tensor(element_type, node_shape, output_base_ptr);
} else {
result.weights = ov::Tensor(element_type, node_shape, data);
}
result.weight_node = std::make_shared<ov::op::v0::Constant>(result.weights);
return result;
}
// Handle quantized weights
if (!ggml_is_quantized(tensor->type)) {
OPENVINO_THROW("Unsupported weight tensor type: ", ggml_type_name(tensor->type));
}
result.layout = ggml_openvino_get_extracted_layout(tensor, use_bias);
const auto & layout = result.layout;
if (layout.total_size == 0) {
OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type));
}
// 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see
// extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul"
// check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly.
// Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose
// zp sizing/type is unaffected by for_gather_matmul, so they are excluded here.
const bool for_gather_matmul = tensor->ne[2] > 1;
const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul);
const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1);
if (is_3d_mxfp4_moe) {
ov::Shape packed_shape = {static_cast<size_t>(tensor->ne[3]),
static_cast<size_t>(tensor->ne[2]),
static_cast<size_t>(tensor->ne[1]),
static_cast<size_t>(tensor->ne[0] / MXFP4_BLOCK_SIZE),
MXFP4_BLOCK_BYTES};
const size_t tensor_bytes = ggml_nbytes(tensor);
if (output_base_ptr) {
auto * buf_base = static_cast<uint8_t *>(output_base_ptr);
memcpy(buf_base + layout.weights_offset, data, tensor_bytes);
result.weights = ov::Tensor(ov::element::u8, packed_shape, buf_base + layout.weights_offset);
} else {
result.weights = ov::Tensor(ov::element::u8, packed_shape);
memcpy(result.weights.data(), data, tensor_bytes);
}
result.weight_node = make_mxfp4_moe_packed_weights(result.weights).get_node_shared_ptr();
result.weight_node->set_friendly_name(tensor->name);
return result;
}
if (use_bias) {
OPENVINO_ASSERT(!layout.is_requant,
"use_bias is only used for test-backend-ops, which should not have requantization");
// bias node will be created on the fly and not use backend buffer
output_base_ptr = nullptr;
}
// F16 requant path - no separate scales/zp needed in result
if (layout.is_requant && layout.requant_type.has_value() && layout.requant_type.value() == ExtraQuantType::F16) {
if (output_base_ptr) {
result.weights = ov::Tensor(ov::element::f16, node_shape,
static_cast<uint8_t *>(output_base_ptr) + layout.weights_offset);
} else {
result.weights = ov::Tensor(ov::element::f16, node_shape);
}
// Not used for F16:
ov::Tensor dummy_scales;
ov::Tensor dummy_zp;
result.weight_node =
requantize_to_buffers(tensor, data, ExtraQuantType::F16, 0, result.weights, dummy_scales, dummy_zp);
return result;
}
// Quantized path (normal extraction or quantized requant)
// Create weight/scale/zp tensors - shared between both paths
// For symmetric quantization, use signed types (i4/i8) and no ZP tensor
ov::element::Type weight_type;
if (tensor->type == GGML_TYPE_MXFP4) {
weight_type = ov::element::f4e2m1;
} else if (layout.is_symmetric) {
weight_type = layout.is_u4 ? ov::element::i4 : ov::element::i8;
} else {
weight_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
}
ov::Shape scale_shape = node_shape;
scale_shape.back() /= layout.weights_per_block;
if (tensor->type == GGML_TYPE_MXFP4) {
if (tensor->ne[2] == 1 && tensor->ne[3] == 1) {
node_shape = {static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
} else {
node_shape.clear();
for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {
node_shape.push_back(static_cast<size_t>(tensor->ne[i]));
}
}
scale_shape = node_shape;
scale_shape.back() /= layout.weights_per_block;
}
const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
if (zp_is_f16) {
zp_type = ov::element::f16;
}
if (output_base_ptr) {
uint8_t * buf_base = static_cast<uint8_t *>(output_base_ptr);
result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset);
result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset);
if (!layout.is_symmetric) {
result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset);
}
// else: result.zp remains default-constructed (empty) for symmetric
} else {
result.weights = ov::Tensor(weight_type, node_shape);
result.scales = ov::Tensor(scale_type, scale_shape);
if (!layout.is_symmetric) {
result.zp = ov::Tensor(zp_type, scale_shape);
}
// else: result.zp remains default-constructed (empty) for symmetric
}
if (layout.is_requant && layout.requant_type.has_value()) {
result.weight_node = requantize_to_buffers(tensor, data, layout.requant_type.value(), layout.weights_per_block,
result.weights, result.scales, result.zp);
} else {
result.weight_node =
extract_quantized_weights(tensor, data, result.weights, result.scales, result.zp, use_bias);
}
return result;
}
ggml/src/ggml-openvino/ggml-quants.h
+1 -139
View File
@@ -2,112 +2,12 @@
#include "ggml-openvino-extra.h" // For ExtraQuantType
#include "ggml.h"
#include <cstdint>
#include <openvino/op/constant.hpp>
#include <openvino/core/node_output.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/runtime/tensor.hpp>
void unpack_32_4(const uint8_t * data, uint8_t * dst);
void extract_q4_0_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr);
void extract_q4_1_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
bool use_bias = false);
void extract_q5_1_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
bool use_bias = false);
void extract_q8_0_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr);
void unpack_256_4(const uint8_t * data, uint8_t * dst);
void extract_q4_k_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
bool use_bias = false);
void extract_q5_k_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
bool use_bias = false);
void extract_q6_k_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr);
void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr);
static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32;
// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights
// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as
// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in
// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see
// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul
// directly from the resulting f16 dequant chain.
//
// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp
// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point);
// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not
// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern.
ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
ov::Tensor & scales,
ov::Tensor & zp,
size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
bool use_bias = false,
bool for_gather_matmul = false);
ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
ov::Tensor & scales,
ov::Tensor & zp,
size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
bool use_bias = false,
bool for_gather_matmul = false);
ov::Output<ov::Node> make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales);
ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight);
// Extract quantized weights from tensor and create weight subgraph
// If weights/scales/zp are provided (non-empty), uses them as output buffers
// Otherwise allocates new ov::Tensors internally
// Returns the weight node (make_int4_weights or make_int8_weights result)
std::shared_ptr<ov::Node> extract_quantized_weights(
const ggml_tensor * tensor,
const void * data, // Source data pointer (may differ from tensor->data)
ov::Tensor & weights,
ov::Tensor & scales,
ov::Tensor & zp,
bool use_bias = false); // Use an exact f16 zero point (vs. a rounded integer one); always
// used for for_gather_matmul (3D MoE expert) weights regardless of
// this flag, and also settable explicitly for test-backend-ops.
// Requantize weights from tensor to target format, writing to provided buffers
// For F16 target, only weights buffer is used (scales/zp ignored)
// Returns the weight node
std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
const void * data, // Source data pointer
ExtraQuantType requant_type,
int64_t block_size,
ov::Tensor & weights,
ov::Tensor & scales,
ov::Tensor & zp);
inline const char * extra_quant_type_name(ExtraQuantType t) {
switch (t) {
case ExtraQuantType::F16:
@@ -156,41 +56,3 @@ OvWeight process_weight_tensor(
// always used for for_gather_matmul (3D MoE expert) weights
// regardless of this flag, and also settable explicitly for
// test-backend-ops.
void quantize_q4_0(const float * x,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
int64_t k,
int64_t qk);
void quantize_q8_1(const float * x,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
int64_t k,
int64_t qk,
int64_t block_offset = 0);
void quantize_q4_1_asym(const float * x,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
int64_t k,
int64_t qk);
void quantize_q8_0(const float * x,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
int64_t k,
int64_t qk,
int64_t block_offset = 0);
namespace ov {
namespace op {
namespace util {
// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp
bool get_single_value(const std::shared_ptr<ov::op::v0::Constant> & const_node,
float & value,
bool check_value_range = true);
} // namespace util
} // namespace op
} // namespace ov
ggml/src/ggml-openvino/model-cache.cpp
+2 -1
View File
@@ -237,7 +237,8 @@ bool ggml_openvino_model_cache_verify_manifest(const std::string & path,
if (!f.is_open()) {
return false;
}
std::string tag, val;
std::string tag;
std::string val;
// header: fingerprint
if (!(f >> tag >> val) || tag != "fingerprint" || val != hex64(fingerprint)) {
return false;
ggml/src/ggml-openvino/openvino/frontend.h
-1
View File
@@ -12,7 +12,6 @@ namespace ggml {
class FrontEnd {
public:
using Ptr = std::shared_ptr<FrontEnd>;
FrontEnd();
static std::shared_ptr<Model> convert(const InputModel::Ptr & model, bool naive = false);
ggml/src/ggml-openvino/openvino/op/add_id.cpp
+1 -1
View File
@@ -20,7 +20,7 @@ namespace op {
static ov::Output<ov::Node> reshape_add_id_input_to_2d(const ov::Output<ov::Node> & input,
const ov::PartialShape & input_shape,
const std::vector<int> & dims) {
const auto actual_shape = input.get_partial_shape();
const auto & actual_shape = input.get_partial_shape();
if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) {
return input;
}
ggml/src/ggml-openvino/openvino/op/cont.cpp
-3
View File
@@ -3,12 +3,9 @@
#include "../op_table.h"
#include "../utils.h"
#include <climits>
#include <cstdint>
#include <memory>
#include <openvino/op/reshape.hpp>
#include <openvino/op/slice.hpp>
#include <vector>
namespace ov {
namespace frontend {
ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
+3 -1
View File
@@ -195,7 +195,9 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output<Node> kv) {
int64_t f = n_heads / n_heads_kv;
if (f > 1 && n_heads_kv > 1) {
ov::Output<ov::Node> kv_broadcast_shape, kv_unsqueezed, new_kv_shape;
ov::Output<ov::Node> kv_broadcast_shape;
ov::Output<ov::Node> kv_unsqueezed;
ov::Output<ov::Node> new_kv_shape;
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2});
kv_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(kv, unsqueeze_axes);
ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
+1 -1
View File
@@ -196,7 +196,7 @@ static OutputVector translate_gated_delta_net_ref(const NodeContext & context) {
}
// Merge batch and head dims: [B*H_v, T, S_v]
auto merge_bh = [&](ov::Output<ov::Node> x, int64_t last_dim) {
auto merge_bh = [&](const ov::Output<ov::Node> & x, int64_t last_dim) {
auto shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{B * H_v, T, last_dim});
return std::make_shared<ov::op::v1::Reshape>(x, shape, false);
};
ggml/src/ggml-openvino/openvino/op/im2col.cpp
-1
View File
@@ -1,7 +1,6 @@
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include "ggml-impl.h"
#include <cstddef>
#include <memory>
ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
+1 -1
View File
@@ -42,7 +42,7 @@ ov::Output<ov::Node> slice_axis(const ov::Output<ov::Node> & input, int64_t axis
ov::Output<ov::Node> static_shape_dims_or_shapeof(const ov::Output<ov::Node> & input,
const std::vector<int> & dims) {
const auto partial_shape = input.get_partial_shape();
const auto & partial_shape = input.get_partial_shape();
if (partial_shape.is_static()) {
std::vector<int64_t> values;
values.reserve(dims.size());
ggml/src/ggml-openvino/openvino/op/pad.cpp
+2 -1
View File
@@ -8,6 +8,7 @@
#include <openvino/op/pad.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/shape_of.hpp>
#include <utility>
#include <vector>
namespace ov {
@@ -20,7 +21,7 @@ namespace {
ov::Output<ov::Node> translate_circular_pad(ov::Output<ov::Node> input,
const std::array<int32_t, 8> & pads,
const ov::Shape & input_shape) {
ov::Output<ov::Node> result = input;
ov::Output<ov::Node> result = std::move(input);
const std::array<int32_t, 4> pads_begin = {pads[6], pads[4], pads[2], pads[0]};
const std::array<int32_t, 4> pads_end = {pads[7], pads[5], pads[3], pads[1]};
ggml/src/ggml-openvino/openvino/op/repeat.cpp
-1
View File
@@ -1,7 +1,6 @@
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include "ggml.h"
#include <memory>
#include <openvino/op/broadcast.hpp>
ggml/src/ggml-openvino/openvino/op/rms_norm.cpp
+1 -3
View File
@@ -25,9 +25,7 @@ OutputVector translate_rms_norm(const NodeContext & context) {
auto op_case = context.get_op_case();
ov::Output<ov::Node> input_node;
if (op_case == 1) {
input_node = process_view_input_new(context, 0);
} else if (op_case == 2) {
if (op_case == 2) {
auto ssm_state_size = context.get_ssm_state_size();
// The GDN op packs [attn | new_state] along the row axis; the state occupies the last
// ssm_state_size * n_seqs rows. Slice it off (scaling by the active sequence count) to keep
ggml/src/ggml-openvino/openvino/op/view.cpp
+2 -2
View File
@@ -7,7 +7,6 @@
#include <openvino/op/reshape.hpp>
#include <openvino/op/shape_of.hpp>
#include <openvino/op/slice.hpp>
#include <set>
namespace ov {
namespace frontend {
@@ -153,7 +152,8 @@ OutputVector translate_view(const NodeContext & context) {
return {input};
}
int64_t src_elems = 1, dst_elems = 1;
int64_t src_elems = 1;
int64_t dst_elems = 1;
for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) {
if (src_shape[i].is_dynamic()) {
return {input};
ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp
+1 -1
View File
@@ -84,7 +84,7 @@ bool KVStateSeqAxis::run_on_model(const std::shared_ptr<ov::Model> & model) {
// Readers still expect seq at dim 1. A reader that is itself the inverse
// Transpose wanted seq at dim 2 all along, so drop it; give anything else the
// inverse Transpose so its input is unchanged.
for (auto & reader : readers) {
for (const auto & reader : readers) {
auto * node = reader.get_node();
if (ov::is_type<ov::op::v6::Assign>(node)) {
continue;
ggml/src/ggml-openvino/openvino/translate_session.cpp
+1 -1
View File
@@ -344,7 +344,7 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
}
};
auto node_visitor = [&](std::shared_ptr<GgmlDecoder> decoder, int node_idx) {
auto node_visitor = [&](const std::shared_ptr<GgmlDecoder> & decoder, int node_idx) {
auto converted_outputs = translate_node(decoder, node_idx);
if (converted_outputs.empty()) {
return;
ggml/src/ggml-openvino/openvino/utils.cpp
+2 -10
View File
@@ -1,7 +1,5 @@
#include "utils.h"
#include "ggml-impl.h"
#include <cmath>
#include <cstddef>
#include <ctime>
@@ -28,13 +26,6 @@ namespace ov {
namespace frontend {
namespace ggml {
std::string getCurrentTime() {
std::time_t now = std::time(nullptr);
char buf[100];
std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now));
return buf;
}
void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs) {
auto input_size = context.get_input_size();
FRONT_END_OP_CONVERSION_CHECK(input_size >= min_inputs, "Got less inputs than expected");
@@ -82,7 +73,7 @@ namespace {
ov::Output<ov::Node> rope_yarn_ramp_mix(int n_dims, const float corr_dims[2], float ext_factor) {
int half_n_dims = n_dims / 2;
std::vector<float> dim_ids_vec(half_n_dims);
std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0);
std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0.0f);
auto dim_ids = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, (size_t) half_n_dims}, dim_ids_vec);
auto corr_low = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[0]});
auto corr_high = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[1]});
@@ -551,6 +542,7 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
if (tail_begin >= 0 && tail_end <= tail_src_elems) {
std::vector<int64_t> flat_shape;
flat_shape.reserve(slice_dim);
for (int i = 0; i < slice_dim; ++i) {
flat_shape.push_back(static_cast<int64_t>(view_src_ggml_shape[i]));
}
ggml/src/ggml-openvino/openvino/utils.h
-2
View File
@@ -14,8 +14,6 @@ namespace ggml {
std::string getCurrentTime();
void dump_ov_model(std::shared_ptr<ov::Model> model);
void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs);
int non_cont_dim(std::vector<size_t> ne, std::vector<size_t> nb);
ggml/src/ggml-openvino/utils.cpp
+461 -473
View File
File diff suppressed because it is too large Load Diff
ggml/src/ggml-openvino/utils.h
-21
View File
@@ -142,9 +142,6 @@ struct ov_runtime_context {
enum ggml_status ov_graph_compute(struct ggml_cgraph * cgraph, ggml_backend_t backend);
enum ggml_status ov_graph_compute_dynamic(struct ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx);
enum ggml_status ov_graph_compute_static(struct ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx);
size_t checksum(const void * data, size_t size);
bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path);
@@ -185,18 +182,6 @@ int64_t get_inp_pos_n_tokens(struct ggml_cgraph * cgraph, const ggml_tensor * in
bool get_is_prefill(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos);
ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name);
ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
const std::string & param_name);
ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
const std::string & param_name,
int chunk_index);
ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
std::shared_ptr<ov::InferRequest> infer_request,
int output_index,
const ggml_tensor * ggml_tensor);
bool is_naive(struct ggml_cgraph * cgraph);
/**
@@ -205,9 +190,3 @@ bool is_naive(struct ggml_cgraph * cgraph);
* @return true if the graph is identified as split; otherwise false.
*/
bool is_model_splitted(struct ggml_cgraph * cgraph);
enum ggml_status naive_compute(struct ggml_cgraph * cgraph,
ov::Core & core,
const std::string & device,
const ov::AnyMap & config,
ov_compiled_model_cache & cache);