server: Don't consider models with --no-mmproj-auto as multimodal (#25590 )

If mmproj is explicitly disabled via the model preset or command-line parameters then the model won't be able to handle image/audio inputs and this shouldn't be declared as supported input modality on the /v1/models endpoint.
mtmd: fix silent prompt truncation on embedded NUL (#25548 )
2026-07-13 16:05:54 +02:00 · 2026-07-13 00:48:13 +02:00 · 2026-07-13 00:47:25 +02:00 · 2026-07-12 15:59:14 -05:00 · 2026-07-12 17:43:51 +02:00 · 2026-07-11 20:03:24 +02:00
10 changed files with 155 additions and 18 deletions
@@ -1081,6 +1081,9 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);
 struct common_prompt_checkpoint {
    int64_t n_tokens;

+    // (optional) id of the task that created the checkpoint
+    int id_task = -1;
+
    llama_pos pos_min;
    llama_pos pos_max;

@@ -313,8 +313,7 @@ llama_model * llama_model_create(llm_arch arch, const llama_model_params & param

    if (model != nullptr) {
        model->arch = arch;
-        auto & devices = model->devices;
-        if (!devices.empty() && devices[0].is_meta && !llm_arch_supports_sm_tensor(arch)) {
+        if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR && !llm_arch_supports_sm_tensor(arch)) {
            throw std::runtime_error(std::string("LLAMA_SPLIT_MODE_TENSOR not implemented for architecture '") + llm_arch_name(arch) + "'");
        }
    }
@@ -250,7 +250,8 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
    LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str());

    mtmd_input_text text;
-    text.text          = formatted_chat.c_str();
+    text.text          = formatted_chat.data();
+    text.text_len      = formatted_chat.size();
    text.add_special   = add_bos;
    text.parse_special = true;

@@ -809,7 +809,7 @@ void mtmd_free(mtmd_context * ctx) {
 struct mtmd_tokenizer {
    mtmd_context * ctx;

-    std::string input_text;
+    std::string input_text; // note: can contain null bytes; do not use c_str()
    bool add_special;
    bool parse_special;
    const llama_vocab * vocab;
@@ -839,9 +839,10 @@ struct mtmd_tokenizer {
            size_t n_bitmaps) : ctx(ctx) {
        add_special   = text->add_special;
        parse_special = text->parse_special;
-        input_text    = text->text;
        vocab         = ctx->vocab;

+        input_text.assign(text->text, text->text_len);
+
        std::vector<const mtmd_bitmap *> bitmaps(bmps, bmps + n_bitmaps);
        auto parts_str = split_text(input_text, ctx->media_marker);
        size_t i_bm = 0;
@@ -67,6 +67,7 @@ struct mtmd_batch;

 struct mtmd_input_text {
    const char * text;
+    size_t text_len;
    bool add_special;
    bool parse_special;
 };
@@ -431,22 +431,70 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                    std::string tool_use_id = json_value(block, "tool_use_id", std::string());

                    auto result_content = json_value(block, "content", json());
-                    std::string result_text;
                    if (result_content.is_string()) {
-                        result_text = result_content.get<std::string>();
+                        tool_results.push_back({
+                            {"role", "tool"},
+                            {"tool_call_id", tool_use_id},
+                            {"content", result_content.get<std::string>()}
+                        });
                    } else if (result_content.is_array()) {
+                        // Single-pass: build both text and content_parts, decide format at the end
+                        std::string result_text;
+                        json content_parts = json::array();
+                        bool has_images = false;
+
                        for (const auto & c : result_content) {
-                            if (json_value(c, "type", std::string()) == "text") {
-                                result_text += json_value(c, "text", std::string());
+                            std::string c_type = json_value(c, "type", std::string());
+                            if (c_type == "text") {
+                                std::string text = json_value(c, "text", std::string());
+                                result_text += text;
+                                content_parts.push_back({
+                                    {"type", "text"},
+                                    {"text", text}
+                                });
+                            } else if (c_type == "image") {
+                                has_images = true;
+                                json source = json_value(c, "source", json::object());
+                                std::string source_type = json_value(source, "type", std::string());
+                                if (source_type == "base64") {
+                                    std::string media_type = json_value(source, "media_type", std::string("image/jpeg"));
+                                    std::string data = json_value(source, "data", std::string());
+                                    std::string url = "data:" + media_type + ";base64," + data;
+                                    content_parts.push_back({
+                                        {"type", "image_url"},
+                                        {"image_url", {{"url", url}}}
+                                    });
+                                } else if (source_type == "url") {
+                                    content_parts.push_back({
+                                        {"type", "image_url"},
+                                        {"image_url", {{"url", json_value(source, "url", std::string())}}}
+                                    });
+                                }
                            }
                        }
-                    }

-                    tool_results.push_back({
-                        {"role", "tool"},
-                        {"tool_call_id", tool_use_id},
-                        {"content", result_text}
-                    });
+                        if (!has_images) {
+                            // Text-only: collapse to a plain string for maximum compatibility
+                            tool_results.push_back({
+                                {"role", "tool"},
+                                {"tool_call_id", tool_use_id},
+                                {"content", result_text}
+                            });
+                        } else {
+                            // Mixed or image-only: use array content parts (OpenAI multimodal tool format)
+                            tool_results.push_back({
+                                {"role", "tool"},
+                                {"tool_call_id", tool_use_id},
+                                {"content", content_parts}
+                            });
+                        }
+                    } else {
+                        tool_results.push_back({
+                            {"role", "tool"},
+                            {"tool_call_id", tool_use_id},
+                            {"content", ""}
+                        });
+                    }
                }
            }

@@ -705,7 +705,8 @@ server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & promp
    std::vector<server_tokens> inputs;
    // multimodal
    mtmd_input_text inp_txt = {
-        prompt.c_str(),
+        prompt.data(),
+        prompt.size(),
        /* add_special */   true,
        /* parse_special */ true,
    };
@@ -2290,6 +2290,24 @@ private:

    // n_tokens_cur: the number of tokens added to the batch for the current slot
    void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) {
+        const int id_task = slot.task->id;
+
+        // evict checkpoints within min-step of a previous checkpoint, unless they were
+        // created by the current task
+        int64_t last = -1;
+        for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) {
+            if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + params_base.checkpoint_min_step) {
+                SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
+                        it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024);
+
+                it = slot.prompt.checkpoints.erase(it);
+                continue;
+            }
+
+            last = it->n_tokens;
+            ++it;
+        }
+
        while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) {
            // make room for the new checkpoint, if needed
            const auto & cur = slot.prompt.checkpoints.front();
@@ -2302,6 +2320,8 @@ private:

        auto & cur = slot.prompt.checkpoints.emplace_back();

+        cur.id_task = id_task;
+
        // [TAG_CHECKPOINTS_FIX_POS_MIN]
        // TODO: here we incorrectly deterimne that the saved checkpoint data covers the [pos_min, pos_max] range
        //       this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225
@@ -3511,7 +3531,10 @@ private:
                    do_checkpoint = do_checkpoint && !has_mtmd;

                    // no need to create checkpoints that are too close together, unless it's the last user message
-                    do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || is_last_user_message || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
+                    do_checkpoint = do_checkpoint && (
+                            slot.prompt.checkpoints.empty() ||
+                            is_last_user_message || near_prompt_end ||
+                            n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
                    SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);

                    // note: we create the checkpoint before calling llama_decode(), so the current batch is not
@@ -219,13 +219,14 @@ void server_model_meta::update_caps() {
            "LLAMA_ARG_MODEL_URL",
            "LLAMA_ARG_MMPROJ",
            "LLAMA_ARG_MMPROJ_URL",
+            "LLAMA_ARG_MMPROJ_AUTO",
            "LLAMA_ARG_HF_REPO",
            "LLAMA_ARG_HF_REPO_FILE",
        });
        params.offline = true;
        common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
        common_models_handler_apply(handler, params); // note: this won't download the model because offline=true
-        if (params.mmproj.path.empty()) {
+        if (params.no_mmproj || params.mmproj.path.empty()) {
            multimodal = { false, false };
        } else {
            multimodal = mtmd_get_cap_from_file(params.mmproj.path.c_str());
@@ -402,6 +402,65 @@ def test_anthropic_tool_result_with_text():
    assert len(res.body["content"]) > 0


+def test_anthropic_tool_result_with_image():
+    """Test tool result containing mixed text and image blocks
+
+    Verifies that image blocks inside Anthropic tool_result content are
+    properly converted to OpenAI image_url format rather than being
+    silently dropped. With a non-multimodal model, the converted image
+    triggers a clear error message instead of being ignored.
+    """
+    server.jinja = True
+    server.start()
+
+    # Small 1x1 red PNG image in base64 (same as vision tests)
+    red_pixel_png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="
+
+    res = server.make_request("POST", "/v1/messages", data={
+        "model": "test",
+        "max_tokens": 100,
+        "messages": [
+            {"role": "user", "content": "What is in this image?"},
+            {
+                "role": "assistant",
+                "content": [
+                    {
+                        "type": "tool_use",
+                        "id": "tool_1",
+                        "name": "read",
+                        "input": {"file": "test.png"}
+                    }
+                ]
+            },
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": "tool_1",
+                        "content": [
+                            {"type": "text", "text": "File: test.png"},
+                            {
+                                "type": "image",
+                                "source": {
+                                    "type": "base64",
+                                    "media_type": "image/png",
+                                    "data": red_pixel_png
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    })
+
+    # Without the fix, image block would cause "unsupported content[].type"
+    # With the fix, image is converted to image_url but tinyllama doesn't support images
+    assert res.status_code == 500
+    assert "image input is not supported" in res.body.get("error", {}).get("message", "").lower()
+
+
 def test_anthropic_tool_result_error():
    """Test tool result with error flag"""
    server.jinja = True
Author	SHA1	Message	Date
Sebastian Dröge	8014d2cf97	server: Don't consider models with --no-mmproj-auto as multimodal (#25590 ) If mmproj is explicitly disabled via the model preset or command-line parameters then the model won't be able to handle image/audio inputs and this shouldn't be declared as supported input modality on the /v1/models endpoint.	2026-07-13 00:48:13 +02:00
Pascal	4114ba18b2	mtmd: fix silent prompt truncation on embedded NUL (#25548 ) * mtmd: fix silent prompt truncation on embedded NUL mtmd_input_text carried the prompt as a bare const char* with no length, so a NUL byte in message content cut the prompt at the tokenizer boundary and dropped every later message plus the assistant marker, with no log. Add an explicit text_len and thread it through, matching llama_tokenize and the text only path. * cleanup --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>	2026-07-13 00:47:25 +02:00
Aldehir Rojas	0c4fa7a989	server : evict checkpoints within min-step of each other (#25472 )	2026-07-12 15:59:14 -05:00
quei	6b4dc2116a	server : fix image blocks in tool_result being dropped during Anthropic OpenAI conversion (#22536 ) * server : fix image blocks in tool_result being dropped during Anthropic→OpenAI conversion server_chat_convert_anthropic_to_oai() silently discarded image blocks inside Anthropic tool_result content. This broke multimodal tool outputs (e.g. a tool that returns an image) because the model never received the image. When tool_result contains image blocks, convert them to OpenAI multimodal content parts (text + image_url array). Plain-text results remain simple strings for backwards compatibility. * server : add test for image blocks in Anthropic tool_result conversion	2026-07-12 17:43:51 +02:00
kdkd	e3546c7948	Fix conditional to display 'LLAMA_SPLIT_MODE_TENSOR not implemented for architecture' message (#24926 )	2026-07-11 20:03:24 +02:00