wylab/llama.cpp
1
0
Fork 1
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-15 00:45:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

server : fix image blocks in tool_result being dropped during Anthropic OpenAI conversion (#22536)

Browse Source 
* server : fix image blocks in tool_result being dropped during Anthropic→OpenAI conversion

server_chat_convert_anthropic_to_oai() silently discarded image blocks

inside Anthropic tool_result content. This broke multimodal tool outputs

(e.g. a tool that returns an image) because the model never received the

image.

When tool_result contains image blocks, convert them to OpenAI

multimodal content parts (text + image_url array). Plain-text results

remain simple strings for backwards compatibility.

* server : add test for image blocks in Anthropic tool_result conversion
This commit is contained in:
quei
2026-07-12 23:43:51 +08:00
committed by GitHub
parent e3546c7948
commit 6b4dc2116a
2 changed files with 117 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-chat.cpp
+58 -10
View File
@@ -431,22 +431,70 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
std::string tool_use_id = json_value(block, "tool_use_id", std::string());
auto result_content = json_value(block, "content", json());
std::string result_text;
if (result_content.is_string()) {
result_text = result_content.get<std::string>();
tool_results.push_back({
{"role", "tool"},
{"tool_call_id", tool_use_id},
{"content", result_content.get<std::string>()}
});
} else if (result_content.is_array()) {
// Single-pass: build both text and content_parts, decide format at the end
std::string result_text;
json content_parts = json::array();
bool has_images = false;
for (const auto & c : result_content) {
if (json_value(c, "type", std::string()) == "text") {
result_text += json_value(c, "text", std::string());
std::string c_type = json_value(c, "type", std::string());
if (c_type == "text") {
std::string text = json_value(c, "text", std::string());
result_text += text;
content_parts.push_back({
{"type", "text"},
{"text", text}
});
} else if (c_type == "image") {
has_images = true;
json source = json_value(c, "source", json::object());
std::string source_type = json_value(source, "type", std::string());
if (source_type == "base64") {
std::string media_type = json_value(source, "media_type", std::string("image/jpeg"));
std::string data = json_value(source, "data", std::string());
std::string url = "data:" + media_type + ";base64," + data;
content_parts.push_back({
{"type", "image_url"},
{"image_url", {{"url", url}}}
});
} else if (source_type == "url") {
content_parts.push_back({
{"type", "image_url"},
{"image_url", {{"url", json_value(source, "url", std::string())}}}
});
}
}
}
}
tool_results.push_back({
{"role", "tool"},
{"tool_call_id", tool_use_id},
{"content", result_text}
});
if (!has_images) {
// Text-only: collapse to a plain string for maximum compatibility
tool_results.push_back({
{"role", "tool"},
{"tool_call_id", tool_use_id},
{"content", result_text}
});
} else {
// Mixed or image-only: use array content parts (OpenAI multimodal tool format)
tool_results.push_back({
{"role", "tool"},
{"tool_call_id", tool_use_id},
{"content", content_parts}
});
}
} else {
tool_results.push_back({
{"role", "tool"},
{"tool_call_id", tool_use_id},
{"content", ""}
});
}
}
}