snapdragon: python SDK setup (Windows) (#27903)

* port setup-build.ps1 to setup_sdk.py, to facilitate installation of Hexagon and OpenCL SDKs on Windows

* rename setup_sdk.py -> setup-sdk.py

* flake8 fix: print() -> logger.info()

---------

Co-authored-by: Kristopher Urquhart <kurquhar@qti.qualcom.com>

common/arg.cpp

@@ -2735,9 +2735,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         "- auto: on, but only for tensors larger than 4 GiB\n"
         "- off: always keep them resident",
         [](common_params & params, const std::string & value) {
-            /**/ if (value == "on")   { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON;   }
-            else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }
-            else if (value == "off")  { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;  }
+            /**/ if (value == "on")   { params.lazy_mode = LLAMA_LAZY_MODE_ON;   }
+            else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
+            else if (value == "off")  { params.lazy_mode = LLAMA_LAZY_MODE_OFF;  }
             else { throw std::invalid_argument("invalid value"); }
         }
     ).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));

common/common.cpp

@@ -1688,7 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
     mparams.main_gpu        = params.main_gpu;
     mparams.split_mode      = params.split_mode;
     mparams.load_mode       = params.load_mode;
-    mparams.tensor_read_lazy = params.tensor_read_lazy;
+    mparams.lazy_mode = params.lazy_mode;
     mparams.tensor_split    = params.tensor_split;
     mparams.check_tensors   = params.check_tensors;
     mparams.use_extra_bufts = !params.no_extra_bufts;

common/common.h

@@ -483,7 +483,7 @@ struct common_params {
     enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
     enum llama_load_mode  load_mode  = LLAMA_LOAD_MODE_AUTO; // how to load the model
 
-    enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; // on-demand reading of tensors marked by the arch
+    enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the arch
 
     common_cpu_params cpuparams;
     common_cpu_params cpuparams_batch;

docs/backend/snapdragon/windows.md

@@ -24,7 +24,18 @@ must be included in the .cat file digitally signed with a trusted certificate.
 This document covers details on how to generate personal certificate files (.pfx) and how to configure the system
 to allow for test signatures (aka test-signing).
 
-## Install the latest Adreno OpenCL SDK
+## Install Windows SDKs
+
+The recommended method is `setup-sdk.py`:
+
+```
+> python scripts\snapdragon\setup-sdk.py --list-sdk-releases
+> python scripts\snapdragon\setup-sdk.py --hexagon --opencl
+```
+
+It installs the selected SDKs under `C:\Qualcomm` and sets their corresponding environment variables for the current user. Start a new terminal after it completes; native Windows builds check all SDK paths before CMake runs.
+
+Select the SDKs to install with `--hexagon` and `--opencl`; use both to prepare a dual-backend build. To select a different available version, pass it to the SDK option, for example `--hexagon 6.4.0.2`. SDK versions install side by side, so you can switch versions without deleting an existing installation. Use `--force` to reinstall the selected SDKs. Use a new CMake build directory after each switch because CMake caches the SDK paths.
 
 Either use the trimmed down version (optimized for CI) from
 

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -17800,20 +17800,32 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
         return;
     }
 
-    auto const &is_empty = [](ggml_tensor * node) -> bool {
+    auto const &is_empty = [](const ggml_tensor * node) -> bool {
         return node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE;
     };
 
-    auto const &is_src_of = [](const ggml_tensor *dst, const ggml_tensor *src) -> bool {
+    auto const &is_src_of = [&is_empty](const ggml_tensor *dst, const ggml_tensor *src) -> bool {
+        auto const &base = [](const ggml_tensor * tensor) {
+            return tensor->view_src ? tensor->view_src : tensor;
+        };
         for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) {
             if (dst->src[s] == src) {
                 return true;
             }
+            if (is_empty(dst) || is_empty(src)) {
+                continue;
+            }
+            // A source view of dst may read storage written through a different view by src.
+            if (dst->src[s] && base(dst->src[s]) == base(src)) {
+                return true;
+            }
+            // Moving dst forward may overwrite storage still read through a view by src.
+            if (src->src[s] && base(dst) == base(src->src[s])) {
+                return true;
+            }
         }
         // implicit dependency if they view the same tensor
-        const ggml_tensor *dst2 = dst->view_src ? dst->view_src : dst;
-        const ggml_tensor *src2 = src->view_src ? src->view_src : src;
-        if (dst2 == src2) {
+        if (base(dst) == base(src)) {
             return true;
         }
         return false;

include/llama.h

@@ -214,10 +214,10 @@ extern "C" {
     LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);
     LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);
 
-    enum llama_tensor_read_lazy {
-        LLAMA_TENSOR_READ_LAZY_OFF  = 0, // always read the whole tensor up front
-        LLAMA_TENSOR_READ_LAZY_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
-        LLAMA_TENSOR_READ_LAZY_ON   = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
+    enum llama_lazy_mode {
+        LLAMA_LAZY_MODE_OFF  = 0, // always read the whole tensor up front
+        LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
+        LLAMA_LAZY_MODE_ON   = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
     };
 
     enum llama_context_type {
@@ -321,7 +321,7 @@ extern "C" {
         enum llama_split_mode split_mode; // how to split the model across multiple GPUs
         enum llama_load_mode  load_mode;  // how to load the model
 
-        enum llama_tensor_read_lazy tensor_read_lazy; // on-demand reading of tensors marked by the arch
+        enum llama_lazy_mode lazy_mode; // on-demand reading of tensors marked by the arch
 
         // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
         int32_t main_gpu;

scripts/snapdragon/build.py

@@ -11,6 +11,8 @@ import platform
 import shutil
 import logging
 
+from sdk import validate_windows_sdks
+
 logger = logging.getLogger("build")
 
 
@@ -65,6 +67,13 @@ def main():
         logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")
         sys.exit(1)
 
+    if target_type == "windows":
+        logger.info("Windows target selected. Forcing native compilation...")
+        args.no_docker = True
+        if platform.system() != "Windows":
+            logger.warning("Warning: Windows compilation is intended to run on Windows arm64 hosts.")
+        validate_windows_sdks()
+
     # Determine preset and check if it's debug
     preset = args.preset
     if preset:
@@ -120,12 +129,6 @@ def main():
 
     jobs = args.jobs if args.jobs else os.cpu_count() or 4
 
-    if target_type == "windows":
-        logger.info("Windows target selected. Forcing native compilation...")
-        args.no_docker = True
-        if platform.system() != "Windows":
-            logger.warning("Warning: Windows compilation is intended to run on Windows arm64 hosts.")
-
     if args.no_docker:
         # Native/local host build
         logger.info("Running native/local CMake build...")
@@ -258,3 +261,6 @@ if __name__ == "__main__":
     except KeyboardInterrupt:
         logger.info("\nInterrupted by user.")
         sys.exit(130)
+    except RuntimeError as err:
+        logger.error("Error: %s", err)
+        sys.exit(1)

scripts/snapdragon/sdk.py

@@ -0,0 +1,62 @@
+import os
+from pathlib import Path
+
+
+SDK_CONFIGS = (
+    {
+        "name": "Hexagon SDK",
+        "repo": "snapdragon-toolchain/hexagon-sdk",
+        "default_version": "6.6.0.0",
+        "parent_dir": "Hexagon_SDK",
+        "archive_prefix": "hexagon-sdk-v",
+        "markers": ("hexagon_sdk.json",),
+    },
+    {
+        "name": "OpenCL SDK",
+        "repo": "snapdragon-toolchain/opencl-sdk",
+        "default_version": "2.3.2",
+        "parent_dir": "OpenCL_SDK",
+        "archive_prefix": "adreno-opencl-sdk-v",
+        "markers": ("include/CL", "lib/OpenCL.lib"),
+    },
+)
+
+
+def is_valid_sdk(config, target_dir):
+    return target_dir.is_dir() and all((target_dir / marker).exists() for marker in config["markers"])
+
+
+def get_hexagon_tools_dir(hexagon_dir):
+    tools_parent = hexagon_dir / "tools" / "HEXAGON_Tools"
+    if not tools_parent.is_dir():
+        raise RuntimeError(f"Expected Hexagon tools directory in {tools_parent}")
+    tools_dirs = [path for path in tools_parent.iterdir() if path.is_dir()]
+    if len(tools_dirs) != 1:
+        raise RuntimeError(f"Expected one Hexagon tools directory in {tools_parent}")
+    return tools_dirs[0]
+
+
+def validate_windows_sdks():
+    hexagon_config, opencl_config = SDK_CONFIGS
+    hexagon_dir = os.environ.get("HEXAGON_SDK_ROOT")
+    tools_dir = os.environ.get("HEXAGON_TOOLS_ROOT")
+    opencl_dir = os.environ.get("OPENCL_SDK_ROOT")
+    missing = []
+
+    expected_tools_dir = None
+    if not hexagon_dir or not is_valid_sdk(hexagon_config, Path(hexagon_dir)):
+        missing.append("HEXAGON_SDK_ROOT")
+    else:
+        try:
+            expected_tools_dir = get_hexagon_tools_dir(Path(hexagon_dir))
+        except RuntimeError:
+            pass
+    if not tools_dir or not expected_tools_dir or Path(tools_dir) != expected_tools_dir:
+        missing.append("HEXAGON_TOOLS_ROOT")
+    if not opencl_dir or not is_valid_sdk(opencl_config, Path(opencl_dir)):
+        missing.append("OPENCL_SDK_ROOT")
+    if missing:
+        raise RuntimeError(
+            f"Missing or invalid Windows SDK paths: {', '.join(missing)}. "
+            "Run scripts/snapdragon/setup-sdk.py first."
+        )

scripts/snapdragon/setup-sdk.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+#
+# Install Windows on Snapdragon SDKs for llama.cpp.
+#
+
+import sys
+import os
+import argparse
+import shutil
+import logging
+import json
+import hashlib
+import tarfile
+import tempfile
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+from sdk import SDK_CONFIGS, get_hexagon_tools_dir, is_valid_sdk
+
+
+logger = logging.getLogger("setup_sdk")
+
+DEFAULT_SDK_BASE_DIR = r"C:\Qualcomm"
+
+
+def get_sdk_releases(config):
+    request = Request(
+        f"https://api.github.com/repos/{config['repo']}/releases?per_page=100",
+        headers={"Accept": "application/vnd.github+json", "User-Agent": "llama.cpp"},
+    )
+    try:
+        with urlopen(request, timeout=30) as response:
+            releases = json.load(response)
+    except (HTTPError, URLError, TimeoutError) as err:
+        raise RuntimeError(f"Cannot query {config['name']} releases: {err}") from err
+
+    result = []
+    for release in releases:
+        if release["draft"] or release["prerelease"]:
+            continue
+        version = release["tag_name"].removeprefix("v")
+        archive_name = f"{config['archive_prefix']}{version}-arm64-wos.tar.xz"
+        for asset in release["assets"]:
+            if asset["name"] != archive_name:
+                continue
+            result.append({
+                "version": version,
+                "name": asset["name"],
+                "url": asset["browser_download_url"],
+                "sha256": (asset.get("digest") or "").removeprefix("sha256:"),
+            })
+    return result
+
+
+def list_sdk_releases():
+    for config in SDK_CONFIGS:
+        logger.info("%s:", config["name"])
+        releases = get_sdk_releases(config)
+        if not releases:
+            logger.info("  no Windows on Snapdragon releases found")
+            continue
+        for release in releases:
+            logger.info("  %s: %s", release["version"], release["name"])
+
+
+def get_sdk_release(config, version):
+    version = version or config["default_version"]
+    version = version.removeprefix("v")
+    for release in get_sdk_releases(config):
+        if release["version"] == version:
+            if not release["sha256"]:
+                raise RuntimeError(f"{config['name']} {version} does not provide a SHA-256 digest")
+            return release
+    raise RuntimeError(
+        f"No Windows on Snapdragon release for {config['name']} {version}. "
+        "Run scripts/snapdragon/setup-sdk.py --list-sdk-releases to see available versions."
+    )
+
+
+def sha256sum(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def download_sdk(release, archive):
+    while True:
+        if archive.exists() and sha256sum(archive) == release["sha256"]:
+            logger.info("Using existing archive %s", archive)
+            return
+
+        offset = archive.stat().st_size if archive.exists() else 0
+        headers = {"User-Agent": "llama.cpp"}
+        if offset:
+            headers["Range"] = f"bytes={offset}-"
+            logger.info("Resuming download of %s at %d MiB", release["name"], offset // (1024 * 1024))
+        else:
+            logger.info("Downloading %s", release["name"])
+
+        try:
+            with urlopen(Request(release["url"], headers=headers), timeout=30) as response:
+                mode = "ab" if offset and response.status == 206 else "wb"
+                with open(archive, mode) as file:
+                    shutil.copyfileobj(response, file)
+        except HTTPError as err:
+            if err.code != 416:
+                raise RuntimeError(f"Cannot download {release['name']}: {err}") from err
+            archive.unlink(missing_ok=True)
+            continue
+        except (URLError, TimeoutError) as err:
+            raise RuntimeError(f"Cannot download {release['name']}: {err}") from err
+
+        if sha256sum(archive) == release["sha256"]:
+            return
+        raise RuntimeError(f"SHA-256 mismatch for {archive}. Re-run the command to resume the download.")
+
+
+def extract_sdk(config, archive, target_dir):
+    if not hasattr(tarfile, "data_filter"):
+        raise RuntimeError("SDK extraction requires Python 3.10.12 or later")
+
+    with tempfile.TemporaryDirectory(prefix=f".{target_dir.name}.tmp-", dir=target_dir.parent) as staging_path:
+        staging_dir = Path(staging_path)
+        with tarfile.open(archive, "r:xz") as tar:
+            tar.extractall(staging_dir, filter=tarfile.data_filter)
+
+        candidates = [staging_dir] + [path for path in staging_dir.iterdir() if path.is_dir()]
+        extracted_dirs = [path for path in candidates if is_valid_sdk(config, path)]
+        if len(extracted_dirs) != 1:
+            raise RuntimeError(f"{config['name']} archive does not contain the expected files")
+        extracted_dir = extracted_dirs[0]
+
+        backup_dir = None
+        if target_dir.exists():
+            backup_dir = target_dir.parent / f".{target_dir.name}.backup"
+            if backup_dir.exists():
+                raise RuntimeError(f"Cannot replace {target_dir}: backup directory {backup_dir} already exists")
+            target_dir.replace(backup_dir)
+        try:
+            extracted_dir.replace(target_dir)
+        except Exception:
+            if backup_dir:
+                backup_dir.replace(target_dir)
+            raise
+        if backup_dir:
+            shutil.rmtree(backup_dir)
+
+
+def install_sdk(config, version, base_dir, force):
+    version = (version or config["default_version"]).removeprefix("v")
+    target_dir = base_dir / config["parent_dir"] / version
+    if is_valid_sdk(config, target_dir) and not force:
+        logger.info("Using existing %s at %s", config["name"], target_dir)
+        return target_dir
+
+    release = get_sdk_release(config, version)
+    target_dir.parent.mkdir(parents=True, exist_ok=True)
+    archive = target_dir.parent / release["name"]
+    download_sdk(release, archive)
+    logger.info("Extracting %s to %s", config["name"], target_dir)
+    extract_sdk(config, archive, target_dir)
+    archive.unlink(missing_ok=True)
+    return target_dir
+
+
+def set_user_environment(values):
+    if os.name != "nt":
+        raise RuntimeError("SDK setup must run on Windows")
+
+    import winreg
+
+    with winreg.CreateKey(winreg.HKEY_CURRENT_USER, "Environment") as key:
+        for name, value in values.items():
+            winreg.SetValueEx(key, name, 0, winreg.REG_SZ, str(value))
+            os.environ[name] = str(value)
+
+    import ctypes
+
+    result = ctypes.c_ulong()
+    ctypes.windll.user32.SendMessageTimeoutW(0xffff, 0x001a, 0, "Environment", 0x0002, 5000, ctypes.byref(result))
+
+
+def setup_sdks(args):
+    base_dir = Path(args.sdk_base_dir).expanduser().resolve()
+    hexagon_config, opencl_config = SDK_CONFIGS
+    environment = {}
+
+    if args.hexagon is not None:
+        hexagon_dir = install_sdk(hexagon_config, args.hexagon, base_dir, args.force)
+        environment["HEXAGON_SDK_ROOT"] = hexagon_dir
+        environment["HEXAGON_TOOLS_ROOT"] = get_hexagon_tools_dir(hexagon_dir)
+    if args.opencl is not None:
+        opencl_dir = install_sdk(opencl_config, args.opencl, base_dir, args.force)
+        environment["OPENCL_SDK_ROOT"] = opencl_dir
+
+    set_user_environment(environment)
+    logger.info("SDK environment variables were updated. Start a new terminal before building.")
+
+
+def main():
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    parser = argparse.ArgumentParser(description="Install Windows on Snapdragon SDKs for llama.cpp.")
+    parser.add_argument("--list-sdk-releases", action="store_true", help="List available Windows on Snapdragon SDK releases")
+    parser.add_argument("--sdk-base-dir", default=DEFAULT_SDK_BASE_DIR, help=r"SDK installation directory (default: C:\Qualcomm)")
+    parser.add_argument("--hexagon", nargs="?", const=SDK_CONFIGS[0]["default_version"], metavar="VERSION", help="Install the Hexagon SDK, optionally selecting a version")
+    parser.add_argument("--opencl", nargs="?", const=SDK_CONFIGS[1]["default_version"], metavar="VERSION", help="Install the OpenCL SDK, optionally selecting a version")
+    parser.add_argument("--force", action="store_true", help="Reinstall selected SDKs even when they already exist")
+    args = parser.parse_args()
+
+    if args.list_sdk_releases:
+        if args.sdk_base_dir != DEFAULT_SDK_BASE_DIR or args.hexagon is not None or args.opencl is not None or args.force:
+            parser.error("Installation options cannot be combined with --list-sdk-releases")
+        list_sdk_releases()
+        return
+    if args.hexagon is None and args.opencl is None:
+        parser.error("Select at least one SDK with --hexagon or --opencl")
+    if os.name != "nt":
+        parser.error("SDK setup must run on Windows")
+    setup_sdks(args)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        logger.info("\nInterrupted by user.")
+        sys.exit(130)
+    except RuntimeError as err:
+        logger.error("Error: %s", err)
+        sys.exit(1)

src/llama-model-loader.cpp

@@ -1287,10 +1287,10 @@ struct ggml_tensor * llama_model_loader::create_tensor(
         return NULL;
     }
 
-    if ((flags & TENSOR_READ_LAZY) && use_mmap && tensor_read_lazy != LLAMA_TENSOR_READ_LAZY_OFF) {
+    if ((flags & TENSOR_READ_LAZY) && use_mmap && lazy_mode != LLAMA_LAZY_MODE_OFF) {
         // in auto mode, small tensors are cheap enough to keep resident
         constexpr size_t auto_lazy_min_size = 4ull * 1024 * 1024 * 1024;
-        if (tensor_read_lazy == LLAMA_TENSOR_READ_LAZY_ON || ggml_nbytes(cur) > auto_lazy_min_size) {
+        if (lazy_mode == LLAMA_LAZY_MODE_ON || ggml_nbytes(cur) > auto_lazy_min_size) {
             const auto & w = require_weight(tn.str().c_str());
             lazy_tensor_ranges[w.idx].emplace_back(w.offs, w.offs + ggml_nbytes(cur));
 

src/llama-model-loader.h

@@ -84,7 +84,7 @@ struct llama_model_loader {
     bool load_mtp;
 
     // set by the caller before the create_tensor() calls
-    enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;
+    enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_OFF;
 
     llama_files files;
     llama_ftype ftype;

src/llama-model.cpp

@@ -2681,7 +2681,7 @@ llama_model_params llama_model_default_params() {
         /*.n_gpu_layers                =*/ -1,
         /*.split_mode                  =*/ LLAMA_SPLIT_MODE_LAYER,
         /*.load_mode                   =*/ LLAMA_LOAD_MODE_AUTO,
-        /*.tensor_read_lazy            =*/ LLAMA_TENSOR_READ_LAZY_AUTO,
+        /*.lazy_mode                   =*/ LLAMA_LAZY_MODE_AUTO,
         /*.main_gpu                    =*/ 0,
         /*.tensor_split                =*/ nullptr,
         /*.progress_callback           =*/ nullptr,

src/llama.cpp

@@ -318,7 +318,7 @@ static std::pair<int, llama_model *> llama_model_load(struct gguf_context * meta
         llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,
             params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides);
 
-        ml.tensor_read_lazy = params.tensor_read_lazy;
+        ml.lazy_mode = params.lazy_mode;
 
         ml.print_info();
         std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, params));

src/models/models.h

@@ -2360,9 +2360,12 @@ struct llama_model_qwen4exp : public llama_model_base {
                         int64_t   channels,
                             int   il);
 
+        ggml_tensor * build_inp_ple(
+  const llama_memory_hybrid_idx_context * mctx_hyb);
+
         ggml_tensor * build_ple(
              llm_graph_input_rs * inp,
-  const llama_memory_hybrid_idx_context * mctx_hyb,
+                    ggml_tensor * emb,
                     ggml_tensor * hidden,
                             int   il);
 

src/models/qwen4exp.cpp

@@ -296,6 +296,7 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
 
     ggml_tensor * inpL = build_inp_embd(model.tok_embd);
     cb(inpL, "model.input_embed", -1);
+    ggml_build_forward_expand(gf, inpL);
 
     auto * inp = build_inp_mem_hybrid();
 
@@ -312,6 +313,13 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
     ggml_tensor * inp_pos     = build_inp_pos();
     ggml_tensor * inp_out_ids = build_inp_out_ids();
 
+    ggml_tensor * ple_emb = nullptr;
+    if (hparams.ple_n_heads > 0) {
+        ple_emb = build_inp_ple(mctx_hyb);
+        // make sure ple_emb and build_inp_embd are in the same graph split
+        ggml_build_forward_expand(gf, ple_emb);
+    }
+
     // the wide residual starts as hc identical copies of the embedding
     ggml_tensor * res_hc = ggml_repeat_4d(ctx0,
             ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens),
@@ -322,7 +330,7 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
         res->t_layer_inp[il] = res_hc;
 
         if (hparams.is_ple(il)) {
-            res_hc = build_ple(inp->get_recr(), mctx_hyb, res_hc, il);
+            res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il);
         }
 
         ggml_tensor * inject = nullptr;
@@ -1090,13 +1098,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at(
     return conv_input;
 }
 
-ggml_tensor * llama_model_qwen4exp::graph::build_ple(
-        llm_graph_input_rs * inp,
-        const llama_memory_hybrid_idx_context * mctx_hyb,
-        ggml_tensor *        hidden,
-        int                  il) {
-    const int64_t hc      = hparams.dsv4_hc_mult;
-    const int64_t hc_dim  = hc * n_embd;
+ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(
+        const llama_memory_hybrid_idx_context * mctx_hyb) {
     const int64_t n_heads = hparams.ple_n_heads;
 
     // the attention cells see every ubatch regardless of the layer types
@@ -1111,7 +1114,18 @@ ggml_tensor * llama_model_qwen4exp::graph::build_ple(
     // gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does
     ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
     emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens);
-    cb(emb, "ple_embd", il);
+    cb(emb, "ple_embd", -1);
+
+    return emb;
+}
+
+ggml_tensor * llama_model_qwen4exp::graph::build_ple(
+        llm_graph_input_rs * inp,
+        ggml_tensor *        emb,
+        ggml_tensor *        hidden,
+        int                  il) {
+    const int64_t hc      = hparams.dsv4_hc_mult;
+    const int64_t hc_dim  = hc * n_embd;
 
     ggml_tensor * key   = build_lora_mm(model.layers[il].ple_key,   emb);
     ggml_tensor * value = build_lora_mm(model.layers[il].ple_value, emb);

tools/llama-bench/README.md

@@ -67,6 +67,7 @@ test parameters:
   -nkvo, --no-kv-offload <0|1>              (default: 0)
   -fa, --flash-attn <on|off|auto>           (default: auto)
   -dev, --device <dev0/dev1/...>            (default: auto)
+  --tensor-read-lazy <on|auto|off>          (default: auto)
   -mmp, --mmap <0|1>                        (DEPRECATED IN FAVOUR OF --load-mode)
   -dio, --direct-io <0|1>                   (DEPRECATED IN FAVOUR OF --load-mode)
   -embd, --embeddings <0|1>                 (default: 0)

tools/llama-bench/llama-bench.cpp

@@ -271,6 +271,19 @@ static const char * split_mode_str(llama_split_mode mode) {
     }
 }
 
+static const char * lazy_mode_str(llama_lazy_mode mode) {
+    switch (mode) {
+        case LLAMA_LAZY_MODE_OFF:
+            return "off";
+        case LLAMA_LAZY_MODE_AUTO:
+            return "auto";
+        case LLAMA_LAZY_MODE_ON:
+            return "on";
+        default:
+            GGML_ABORT("invalid tensor read lazy mode");
+    }
+}
+
 static std::string pair_str(const std::pair<int, int> & p) {
     static char buf[32];
     snprintf(buf, sizeof(buf), "%d,%d", p.first, p.second);
@@ -341,6 +354,7 @@ struct cmd_params {
     std::vector<int>                 n_cpu_moe;
     std::vector<llama_split_mode>    split_mode;
     std::vector<llama_load_mode>     load_mode;
+    std::vector<llama_lazy_mode>     lazy_mode;
     std::vector<int>                 main_gpu;
     std::vector<bool>                no_kv_offload;
     std::vector<llama_flash_attn_type> flash_attn;
@@ -385,6 +399,7 @@ static const cmd_params cmd_params_defaults = {
     /* n_cpu_moe            */ { 0 },
     /* split_mode           */ { LLAMA_SPLIT_MODE_LAYER },
     /* load_mode            */ { LLAMA_LOAD_MODE_AUTO },
+    /* lazy_mode            */ { LLAMA_LAZY_MODE_AUTO },
     /* main_gpu             */ { 0 },
     /* no_kv_offload        */ { false },
     /* flash_attn           */ { LLAMA_FLASH_ATTN_TYPE_AUTO },
@@ -460,6 +475,7 @@ static void print_usage(int /* argc */, char ** argv) {
     printf("  -fa, --flash-attn <on|off|auto>                   (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
     printf("  -dev, --device <dev0/dev1/...>                    (default: auto)\n");
     printf("  -lm, --load-mode <auto|none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
+    printf("  --tensor-read-lazy <on|auto|off>                  (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str());
     printf("  -mmp, --mmap <0|1>                                (DEPRECATED IN FAVOUR OF --load-mode)\n");
     printf("  -dio, --direct-io <0|1>                           (DEPRECATED IN FAVOUR OF --load-mode)\n");
     printf("  -embd, --embeddings <0|1>                         (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
@@ -786,6 +802,32 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
                     break;
                 }
                 params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());
+            } else if (arg == "--tensor-read-lazy") {
+                if (++i >= argc) {
+                    invalid_param = true;
+                    break;
+                }
+                auto p = string_split<std::string>(argv[i], split_delim);
+
+                std::vector<llama_lazy_mode> modes;
+                for (const auto & m : p) {
+                    llama_lazy_mode mode;
+                    if (m == "on") {
+                        mode = LLAMA_LAZY_MODE_ON;
+                    } else if (m == "auto") {
+                        mode = LLAMA_LAZY_MODE_AUTO;
+                    } else if (m == "off") {
+                        mode = LLAMA_LAZY_MODE_OFF;
+                    } else {
+                        invalid_param = true;
+                        break;
+                    }
+                    modes.push_back(mode);
+                }
+                if (invalid_param) {
+                    break;
+                }
+                params.lazy_mode.insert(params.lazy_mode.end(), modes.begin(), modes.end());
             } else if (arg == "-mg" || arg == "--main-gpu") {
                 if (++i >= argc) {
                     invalid_param = true;
@@ -1137,6 +1179,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
     if (params.load_mode.empty()) {
         params.load_mode = cmd_params_defaults.load_mode;
     }
+    if (params.lazy_mode.empty()) {
+        params.lazy_mode = cmd_params_defaults.lazy_mode;
+    }
     if (params.main_gpu.empty()) {
         params.main_gpu = cmd_params_defaults.main_gpu;
     }
@@ -1203,6 +1248,7 @@ struct cmd_params_instance {
     int                n_cpu_moe;
     llama_split_mode   split_mode;
     llama_load_mode    load_mode;
+    llama_lazy_mode    lazy_mode;
     int                main_gpu;
     bool               no_kv_offload;
     llama_flash_attn_type flash_attn;
@@ -1224,6 +1270,7 @@ struct cmd_params_instance {
         }
         mparams.split_mode    = split_mode;
         mparams.load_mode     = load_mode;
+        mparams.lazy_mode     = lazy_mode;
         mparams.main_gpu      = main_gpu;
         mparams.tensor_split  = tensor_split.data();
         mparams.no_host       = no_host;
@@ -1271,7 +1318,8 @@ struct cmd_params_instance {
         return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe &&
                split_mode == other.split_mode &&
                main_gpu == other.main_gpu && tensor_split == other.tensor_split &&
-               load_mode == other.load_mode && devices == other.devices && no_host == other.no_host &&
+               load_mode == other.load_mode && lazy_mode == other.lazy_mode &&
+               devices == other.devices && no_host == other.no_host &&
                vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);
     }
 
@@ -1305,6 +1353,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
     for (const auto & ncmoe : params.n_cpu_moe)
     for (const auto & sm : params.split_mode)
     for (const auto & lm : params.load_mode)
+    for (const auto & lzm : params.lazy_mode)
     for (const auto & mg : params.main_gpu)
     for (const auto & devs : params.devices)
     for (const auto & ts : params.tensor_split)
@@ -1344,6 +1393,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
                 /* .n_cpu_moe             = */ ncmoe,
                 /* .split_mode            = */ sm,
                 /* .load_mode             = */ lm,
+                /* .lazy_mode             = */ lzm,
                 /* .main_gpu              = */ mg,
                 /* .no_kv_offload         = */ nkvo,
                 /* .flash_attn            = */ fa,
@@ -1380,6 +1430,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
                 /* .n_cpu_moe             = */ ncmoe,
                 /* .split_mode            = */ sm,
                 /* .load_mode             = */ lm,
+                /* .lazy_mode             = */ lzm,
                 /* .main_gpu              = */ mg,
                 /* .no_kv_offload         = */ nkvo,
                 /* .flash_attn            = */ fa,
@@ -1416,6 +1467,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
                 /* .n_cpu_moe             = */ ncmoe,
                 /* .split_mode            = */ sm,
                 /* .load_mode             = */ lm,
+                /* .lazy_mode             = */ lzm,
                 /* .main_gpu              = */ mg,
                 /* .no_kv_offload         = */ nkvo,
                 /* .flash_attn            = */ fa,
@@ -1457,6 +1509,7 @@ struct test {
     int                      n_cpu_moe;
     llama_split_mode         split_mode;
     llama_load_mode          load_mode;
+    llama_lazy_mode          lazy_mode;
     int                      main_gpu;
     bool                     no_kv_offload;
     llama_flash_attn_type    flash_attn;
@@ -1496,6 +1549,7 @@ struct test {
         n_cpu_moe      = inst.n_cpu_moe;
         split_mode     = inst.split_mode;
         load_mode      = inst.load_mode;
+        lazy_mode      = inst.lazy_mode;
         main_gpu       = inst.main_gpu;
         no_kv_offload  = inst.no_kv_offload;
         flash_attn     = inst.flash_attn;
@@ -1563,7 +1617,8 @@ struct test {
             "n_ubatch",       "n_threads",      "cpu_mask",      "cpu_strict",     "poll",
             "type_k",         "type_v",         "n_gpu_layers",  "n_cpu_moe",      "split_mode",
             "main_gpu",       "no_kv_offload",  "flash_attn",    "devices",        "tensor_split",
-            "tensor_buft_overrides",            "load_mode",     "embeddings",
+            "tensor_buft_overrides",            "load_mode",     "lazy_mode",
+            "embeddings",
             "no_op_offload",  "no_host",        "fit_target",    "fit_min_ctx",
             "n_prompt",       "n_gen",          "n_depth",
             "test_time",      "avg_ns",         "stddev_ns",     "avg_ts",         "stddev_ts"
@@ -1588,7 +1643,7 @@ struct test {
         if (field == "avg_ts" || field == "stddev_ts") {
             return FLOAT;
         }
-        if (field == "load_mode") {
+        if (field == "load_mode" || field == "lazy_mode") {
             return STRING;
         }
         return STRING;
@@ -1658,6 +1713,7 @@ struct test {
                                             tensor_split_str,
                                             tensor_buft_overrides_str,
                                             llama_load_mode_name(load_mode),
+                                            lazy_mode_str(lazy_mode),
                                             std::to_string(embeddings),
                                             std::to_string(no_op_offload),
                                             std::to_string(no_host),
@@ -1972,6 +2028,9 @@ struct markdown_printer : public printer {
         if (params.load_mode.size() > 1 || params.load_mode != cmd_params_defaults.load_mode) {
             fields.emplace_back("load_mode");
         }
+        if (params.lazy_mode.size() > 1 || params.lazy_mode != cmd_params_defaults.lazy_mode) {
+            fields.emplace_back("lazy_mode");
+        }
         if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) {
             fields.emplace_back("embeddings");
         }