wylab/llama.cpp
0
0
Fork 1
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-14 18:02:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

server: cleanup (remove slot.batch_spec, rename)

Browse Source
This commit is contained in:
Sascha Rogmann
2026-01-24 15:55:02 +01:00
parent 1e29af4ea5
commit a1584ac80f
11 changed files with 41 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+4 -3
View File
@@ -629,7 +629,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
{
bool has_draft =!params.speculative.model.path.empty();
bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3
bool has_lookup_caches = !params.lookup_cache_static.empty() && !params.lookup_cache_dynamic.empty();
bool has_lookup_caches = !params.speculative.lookup_cache_static.empty()
&& !params.speculative.lookup_cache_dynamic.empty();
bool has_simple = (params.speculative.draftless_type == COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE);
bool found_config_draft = false;
bool found_config_eagle3 = false;
@@ -1253,14 +1254,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
{"-lcs", "--lookup-cache-static"}, "FNAME",
"path to static lookup cache to use for lookup decoding (not updated by generation)",
[](common_params & params, const std::string & value) {
params.lookup_cache_static = value;
params.speculative.lookup_cache_static = value;
}
).set_examples({LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"-lcd", "--lookup-cache-dynamic"}, "FNAME",
"path to dynamic lookup cache to use for lookup decoding (updated by generation)",
[](common_params & params, const std::string & value) {
params.lookup_cache_dynamic = value;
params.speculative.lookup_cache_dynamic = value;
}
).set_examples({LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
common/common.h
+7 -4
View File
@@ -281,11 +281,16 @@ struct common_params_speculative {
struct common_params_model model;
// draftless:
common_speculative_type draftless_type = COMMON_SPECULATIVE_TYPE_NONE; // type of speculative decoding without a draft model
uint16_t spec_ngram_size_n = 12;
uint16_t spec_ngram_size_m = 48;
uint16_t spec_ngram_size_n = 12; // ngram size for lookup
uint16_t spec_ngram_size_m = 48; // mgram size for speculative tokens
std::vector<common_speculative_config> configs = {}; // list of speculative configs to try
std::string lookup_cache_static = ""; // path of static ngram cache file for lookup decoding // NOLINT
std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding // NOLINT
};
struct common_params_vocoder {
@@ -403,8 +408,6 @@ struct common_params {
std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state // NOLINT
std::string input_prefix = ""; // string to prefix user inputs with // NOLINT
std::string input_suffix = ""; // string to suffix user inputs with // NOLINT
std::string lookup_cache_static = ""; // path of static ngram cache file for lookup decoding // NOLINT
std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding // NOLINT
std::string logits_file = ""; // file for saving *all* logits // NOLINT
// llama-debug specific options
common/ngram-map.cpp
+1 -1
View File
@@ -237,7 +237,7 @@ void common_ngram_map_draft(common_ngram_map & map,
map.last_draft_value_idx = slot_max; // value used for draft generation.
}
void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted) {
void common_ngram_map_accept(common_ngram_map & map, uint16_t n_accepted) {
if (!map.last_draft_created) {
return;
}
common/ngram-map.h
+2 -2
View File
@@ -62,5 +62,5 @@ void common_ngram_map_draft(
const llama_tokens & inp, llama_token sampled,
llama_tokens & draft);
// Update the statistics of a value after a draft was accepted.
void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted);
// Update the statistics of a value after a draft was processed.
void common_ngram_map_accept(common_ngram_map & map, uint16_t n_accepted);
common/speculative.cpp
+7 -7
View File
@@ -251,12 +251,12 @@ enum common_speculative_type common_speculative_type_from_name(const std::string
struct common_speculative * common_speculative_init(
struct common_params & params,
struct common_params_speculative & params,
struct llama_context * ctx_tgt,
struct llama_context * ctx_dft
) {
std::vector<std::unique_ptr<common_speculative_state>> implementations = {};
for (const common_speculative_config & config : params.speculative.configs) {
for (const common_speculative_config & config : params.configs) {
LOG_INF("common_speculative_init: adding implementation %s\n", common_speculative_type_to_str(config.type).c_str());
switch (config.type) {
case COMMON_SPECULATIVE_TYPE_NONE:
@@ -271,7 +271,7 @@ struct common_speculative * common_speculative_init(
}
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {
common_ngram_map ngram_map = get_common_ngram_map(config,
params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m);
params.spec_ngram_size_n, params.spec_ngram_size_m);
uint16_t ngram_size_key = ngram_map.size_key;
uint16_t mgram_size_value = ngram_map.size_value;
uint16_t check_rate = ngram_map.check_rate;
@@ -287,14 +287,14 @@ struct common_speculative * common_speculative_init(
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: {
implementations.push_back(std::make_unique<common_speculative_state_ngram_map_k>(
(config.type), get_common_ngram_map(config,
params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m)
params.spec_ngram_size_n, params.spec_ngram_size_m)
));
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: {
implementations.push_back(std::make_unique<common_speculative_state_ngram_map_k4v>(
(config.type), get_common_ngram_map(config,
params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m)));
params.spec_ngram_size_n, params.spec_ngram_size_m)));
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: {
@@ -746,7 +746,7 @@ llama_tokens common_speculative_use_draft_model(
return result;
}
void common_speculative_send_accepted(struct common_speculative * spec, const uint16_t n_accepted) {
void common_speculative_accept(struct common_speculative * spec, const uint16_t n_accepted) {
common_speculative_state * impl = spec->curr_impl;
if (impl != nullptr) {
if (n_accepted > 0) {
@@ -756,7 +756,7 @@ void common_speculative_send_accepted(struct common_speculative * spec, const ui
if (impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K ||
impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V) {
auto state = static_cast<struct common_speculative_state_ngram_map_k *>(impl);
common_ngram_map_send_accepted(state->map, n_accepted);
common_ngram_map_accept(state->map, n_accepted);
}
}
}
common/speculative.h
+2 -2
View File
@@ -22,7 +22,7 @@ enum common_speculative_type common_speculative_type_from_name(const std::string
std::string common_speculative_type_to_str(enum common_speculative_type type);
struct common_speculative * common_speculative_init(
struct common_params & params,
struct common_params_speculative & params,
struct llama_context * ctx_tgt,
struct llama_context * ctx_dft
);
@@ -45,7 +45,7 @@ llama_tokens common_speculative_gen_draft(
llama_token id_last);
// informs the speculative decoder that n_accepted tokens were accepted by the target model
void common_speculative_send_accepted(
void common_speculative_accept(
struct common_speculative * spec,
const uint16_t n_accepted);
examples/lookup/lookup-create.cpp
+2 -2
View File
@@ -32,9 +32,9 @@ int main(int argc, char ** argv){
common_ngram_cache ngram_cache;
common_ngram_cache_update(ngram_cache, LLAMA_NGRAM_STATIC, LLAMA_NGRAM_STATIC, inp, inp.size(), true);
fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.lookup_cache_static.c_str());
fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.speculative.lookup_cache_static.c_str());
common_ngram_cache_save(ngram_cache, params.lookup_cache_static);
common_ngram_cache_save(ngram_cache, params.speculative.lookup_cache_static);
return 0;
}
examples/lookup/lookup-stats.cpp
+5 -5
View File
@@ -46,18 +46,18 @@ int main(int argc, char ** argv){
{
const int64_t t_start_draft_us = ggml_time_us();
if (!params.lookup_cache_static.empty()) {
if (!params.speculative.lookup_cache_static.empty()) {
try {
ngram_cache_static = common_ngram_cache_load(params.lookup_cache_static);
ngram_cache_static = common_ngram_cache_load(params.speculative.lookup_cache_static);
} catch (std::ifstream::failure const &) {
LOG_ERR("failed to open static lookup cache: %s", params.lookup_cache_static.c_str());
LOG_ERR("failed to open static lookup cache: %s", params.speculative.lookup_cache_static.c_str());
exit(1);
}
}
if (!params.lookup_cache_dynamic.empty()) {
if (!params.speculative.lookup_cache_dynamic.empty()) {
try {
ngram_cache_dynamic = common_ngram_cache_load(params.lookup_cache_dynamic);
ngram_cache_dynamic = common_ngram_cache_load(params.speculative.lookup_cache_dynamic);
} catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program
}
examples/lookup/lookup.cpp
+6 -6
View File
@@ -51,18 +51,18 @@ int main(int argc, char ** argv){
const int64_t t_start_draft_us = ggml_time_us();
common_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, inp, inp.size(), false);
if (!params.lookup_cache_static.empty()) {
if (!params.speculative.lookup_cache_static.empty()) {
try {
ngram_cache_static = common_ngram_cache_load(params.lookup_cache_static);
ngram_cache_static = common_ngram_cache_load(params.speculative.lookup_cache_static);
} catch (std::ifstream::failure const &) {
LOG_ERR("failed to open static lookup cache: %s", params.lookup_cache_static.c_str());
LOG_ERR("failed to open static lookup cache: %s", params.speculative.lookup_cache_static.c_str());
exit(1);
}
}
if (!params.lookup_cache_dynamic.empty()) {
if (!params.speculative.lookup_cache_dynamic.empty()) {
try {
ngram_cache_dynamic = common_ngram_cache_load(params.lookup_cache_dynamic);
ngram_cache_dynamic = common_ngram_cache_load(params.speculative.lookup_cache_dynamic);
} catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program
}
@@ -210,7 +210,7 @@ int main(int argc, char ** argv){
// Update dynamic ngram cache with context ngram cache and save it to disk:
common_ngram_cache_merge(ngram_cache_dynamic, ngram_cache_context);
common_ngram_cache_save(ngram_cache_dynamic, params.lookup_cache_dynamic);
common_ngram_cache_save(ngram_cache_dynamic, params.speculative.lookup_cache_dynamic);
LOG("\n\n");
examples/speculative-simple/speculative-simple.cpp
+1 -1
View File
@@ -132,7 +132,7 @@ int main(int argc, char ** argv) {
params_spec.n_reuse = llama_n_ctx(ctx_dft) - n_draft;
params_spec.p_min = p_min;
struct common_speculative * spec = common_speculative_init(params, ctx_tgt, ctx_dft);
struct common_speculative * spec = common_speculative_init(params.speculative, ctx_tgt, ctx_dft);
for (auto &pair : params.speculative.replacements) {
common_speculative_add_replacement_tgt_dft(spec, pair.first.c_str(), pair.second.c_str());
}
tools/server/server-context.cpp
+4 -18
View File
@@ -48,8 +48,6 @@ enum server_state {
struct server_slot {
int id;
llama_batch batch_spec = {};
// TODO: change to unique_ptrs for consistency:
llama_context * ctx = nullptr;
llama_context * ctx_dft = nullptr;
@@ -604,8 +602,6 @@ private:
common_speculative_free(slot.spec);
slot.spec = nullptr;
llama_batch_free(slot.batch_spec);
}
llama_batch_free(batch);
@@ -766,8 +762,6 @@ private:
slot.prompt.tokens.has_mtmd = mctx != nullptr;
if (model_dft) {
slot.batch_spec = llama_batch_init(params_base.speculative.n_max + 1, 0, 1);
// TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK]
slot.ctx_dft = llama_init_from_model(model_dft, cparams_dft);
if (slot.ctx_dft == nullptr) {
@@ -775,7 +769,7 @@ private:
return false;
}
slot.spec = common_speculative_init(params_base, slot.ctx, slot.ctx_dft);
slot.spec = common_speculative_init(params_base.speculative, slot.ctx, slot.ctx_dft);
if (slot.spec == nullptr) {
SRV_ERR("%s", "failed to create speculator\n");
return false;
@@ -784,7 +778,7 @@ private:
common_speculative_add_replacement_tgt_dft(slot.spec, pair.first.c_str(), pair.second.c_str());
}
} else if (params_base.speculative.configs.size() > 0) {
slot.spec = common_speculative_init(params_base, nullptr, nullptr);
slot.spec = common_speculative_init(params_base.speculative, nullptr, nullptr);
}
SLT_INF(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
@@ -1185,14 +1179,6 @@ private:
slot.task = std::make_unique<const server_task>(std::move(task));
// initialize draft batch
// TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK]
if (slot.can_speculate()) {
llama_batch_free(slot.batch_spec);
slot.batch_spec = llama_batch_init(task.params.speculative.n_max + 1, 0, 1);
}
slot.state = slot.task->is_child()
? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
: SLOT_STATE_STARTED;
@@ -2817,8 +2803,8 @@ private:
// update how many tokens out of those tested were accepted
slot.n_draft_accepted += ids.size() - 1;
// inform the speculative decoding about the accepted tokens
common_speculative_send_accepted(slot.spec, ids.size() - 1);
// inform the speculative decoding about the number of accepted tokens
common_speculative_accept(slot.spec, ids.size() - 1);
// rollback to the state before sampling the draft tokens
slot.prompt.tokens.keep_first(slot.prompt.n_tokens() - n_draft);