Compare commits

...

3 Commits

Author SHA1 Message Date
Alessandro de Oliveira Faria (A.K.A.CABELO) e2f59ed71d vendor: update BoringSSL to 0.20260728.0 (#26241) 2026-07-29 15:16:02 +03:00
Georgi Gerganov 992c325323 server : add trace logging for slot similarity checking (#26271)
Adds trace logging in server-context.cpp for slot similarity checking
during prompt cache slot selection, including skip reasons and similarity
calculation details.

Assisted-by: llama.cpp:Qwen3.6-27B
2026-07-29 14:59:44 +03:00
Kaben Nanlohy e1af89a681 conversion: fix Qwen2.5-Omni mmproj conversion regression (#26262) 2026-07-29 12:53:44 +02:00
4 changed files with 23 additions and 18 deletions
+3 -3
View File
@@ -179,12 +179,12 @@ class Qwen25OmniModel(Qwen2VLVisionModel, Qwen25AudioModel):
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
if not name.startswith("visual.") and not name.startswith("audio_tower."):
return None
if name.startswith("thinker."):
name = name.replace("thinker.", "")
if not name.startswith("visual.") and not name.startswith("audio_tower."):
return None
if "audio_bos_eos_token" in name:
# this tensor is left unused in transformers code
# https://github.com/huggingface/transformers/blob/6e3063422c4b1c014aa60c32b9254fd2902f0f28/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py#L1809
+12 -7
View File
@@ -1537,7 +1537,7 @@ private:
// find the slot that has at least n% prompt similarity
if (slot_prompt_similarity != 0.0f) {
float sim_best = 0;
float f_sim_best = 0;
for (server_slot & slot : slots) {
if (task.id_slot != -1 && slot.id != task.id_slot) {
@@ -1546,6 +1546,7 @@ private:
// skip the slot if it is not available
if (slot.is_processing()) {
SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing());
continue;
}
@@ -1553,26 +1554,30 @@ private:
// skip the slot if it does not contains cached tokens
if (tokens.empty()) {
SLT_TRC(slot, "%s", " - skipping, slot is empty\n");
continue;
}
// fraction of the Longest Common Prefix length with respect to the input prompt length
const float sim_cur = float(tokens.get_common_prefix(task.tokens)) / task.tokens.size();
const size_t lcp_len = tokens.get_common_prefix(task.tokens);
const float f_sim_cur = float(lcp_len) / task.tokens.size();
SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);
// select the current slot if the criteria match
if (sim_cur > sim_best && sim_cur > slot_prompt_similarity) {
sim_best = sim_cur;
if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {
f_sim_best = f_sim_cur;
ret = &slot;
}
}
if (ret != nullptr) {
const float f_keep = (sim_best*task.tokens.size()) / ret->prompt.tokens.size();
const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt.tokens.size();
if (task.id_slot == -1) {
SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
sim_best, slot_prompt_similarity, f_keep);
SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
f_sim_best, slot_prompt_similarity, f_keep);
}
// if we are about to lose a large portion of the existing context - save it in the prompt cache
+7 -7
View File
@@ -1742,9 +1742,9 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
float sim_best = float(lcp_best) / tokens_new.size();
float f_sim_best = float(lcp_best) / tokens_new.size();
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
auto it_best = states.end();
@@ -1753,25 +1753,25 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new);
const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size();
const float sim_cur = float(lcp_cur) / tokens_new.size();
const float f_sim_cur = float(lcp_cur) / tokens_new.size();
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, sim_cur);
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
// don't trash large prompts
if (f_keep_cur < 0.25f) {
continue;
}
if (f_keep_best < f_keep_cur && sim_best < sim_cur) {
if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
f_keep_best = f_keep_cur;
sim_best = sim_cur;
f_sim_best = f_sim_cur;
it_best = it;
}
}
if (it_best != states.end()) {
SRV_TRC(" - found better prompt with f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
{
auto & data = it_best->data.main;
+1 -1
View File
@@ -41,7 +41,7 @@ if (LLAMA_BUILD_BORINGSSL)
set(FIPS OFF CACHE BOOL "Enable FIPS (BoringSSL)")
set(BORINGSSL_GIT "https://boringssl.googlesource.com/boringssl" CACHE STRING "BoringSSL git repository")
set(BORINGSSL_VERSION "0.20260713.0" CACHE STRING "BoringSSL version")
set(BORINGSSL_VERSION "0.20260728.0" CACHE STRING "BoringSSL version")
message(STATUS "Fetching BoringSSL version ${BORINGSSL_VERSION}")