mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-06-09 07:16:44 +02:00
server: expose prompt token counts in /slots endpoint (#23454)
Add n_prompt_tokens, n_prompt_tokens_processed, and n_prompt_tokens_cache to the /slots JSON response. These fields are already tracked internally but were not exposed, making it impossible for clients to monitor prompt evaluation progress during processing.
This commit is contained in:
@@ -506,6 +506,9 @@ struct server_slot {
|
||||
|
||||
if (ptask) {
|
||||
res["id_task"] = ptask->id;
|
||||
res["n_prompt_tokens"] = (int32_t) prompt.tokens.size();
|
||||
res["n_prompt_tokens_processed"] = n_prompt_tokens_processed;
|
||||
res["n_prompt_tokens_cache"] = n_prompt_tokens_cache;
|
||||
res["params"] = ptask->params.to_json(only_metrics);
|
||||
res["next_token"] = {
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user