nits

common/arg.cpp

@@ -3757,6 +3757,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.sleep_idle_seconds = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--sleep-mode"}, "MODE",
+        "sleep behavior:\n"
+        "- 'free' frees context and model memory\n"
+        "- 'rst' restarts the whole process, may help reset memory to zero on certain backend (only support posix env)\n"
+        "(default: free)",
+        [](common_params & params, const std::string & value) {
+            if (value == "free") {
+                params.sleep_mode = COMMON_SLEEP_MODE_FREE;
+            } else if (value == "rst") {
+                params.sleep_mode = COMMON_SLEEP_MODE_RST;
+            } else {
+                throw std::invalid_argument("invalid value: " + value);
+            }
+        }
+    ).set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"--simple-io"},
         "use basic IO for better compatibility in subprocesses and limited consoles",

common/common.h

@@ -408,6 +408,11 @@ struct common_params_diffusion {
 
 // reasoning API response format (not to be confused as chat template's reasoning format)
 // only used by server
+enum common_sleep_mode {
+    COMMON_SLEEP_MODE_FREE, // free context and model memory
+    COMMON_SLEEP_MODE_RST, // also restart the process, releasing all backend resources
+};
+
 enum common_reasoning_format {
     COMMON_REASONING_FORMAT_NONE,
     COMMON_REASONING_FORMAT_AUTO,            // Same as deepseek, using `message.reasoning_content`
@@ -633,6 +638,7 @@ struct common_params {
     int enable_reasoning = -1; // -1 = auto, 0 = disable, 1 = enable
     bool prefill_assistant = true; // if true, any trailing assistant message will be prefilled into the response
     int sleep_idle_seconds = -1;   // if >0, server will sleep after this many seconds of idle time
+    common_sleep_mode sleep_mode = COMMON_SLEEP_MODE_FREE;
 
     std::vector<std::string> api_keys;
 

tools/server/README-dev.md

@@ -321,6 +321,25 @@ Call stack on waking up:
 
 Endpoints created with `create_response(true)` (`/health`, `/props`, `/models`, `/metrics`) skip `wait_until_no_sleep`, so they answer from the cached responses instead of waking the server.
 
+#### Process reset (`--sleep-mode rst`)
+
+As described in the section above, the sleeping function frees the `llama_context` and `llama_model` instances. However, on certain backends, the underlying driver or library still leaves behind some residual memory. See issue: [#19379](https://github.com/ggml-org/llama.cpp/issues/19379), [#25570](https://github.com/ggml-org/llama.cpp/issues/25570)
+
+Multiple PRs attempted to fix the problem by simply exiting the process ([#25243](https://github.com/ggml-org/llama.cpp/pull/25243), [#27307](https://github.com/ggml-org/llama.cpp/pull/27307)). However, the main issues with this approach are: (1) it requires router mode to handle the respawn, (2) `/props` and `/models` cannot be accessed during sleep and (3) metrics are reset.
+
+Note that [#25271](https://github.com/ggml-org/llama.cpp/pull/25271) proposed a deeper solution, adding a GGML API to reset the physical device. However, it does not work correctly on AMD GPUs due to a limitation of the AMD Tensile library, and the CUDA backend keeps static state that assumes the context is never destroyed.
+
+Therefore, `--sleep-mode rst` was added to reset the process instead (PR [#27418](https://github.com/ggml-org/llama.cpp/pull/27418)), while still allowing `/props` and `/models` to work as-is, and keeping the metrics. Only POSIX platforms are supported for now, because this relies on `exec()` to re-use the same PID. Windows is not yet supported and may require another method.
+
+The way it works: The restart is handled by `server_sleep_rst`, owned by `server_context_impl`:
+- `restart()` is called by `handle_sleeping_state()` right after `destroy()`, so the model is already unloaded
+- the state to preserve is the cached responses (`server_routes::cache_to_json`), passed to the new process via the `LLAMA_SERVER_SLEEP_STATE` env var
+- `init()` reads that env var and clears it, so that child processes do not inherit it
+- all file descriptors except stdio are marked `FD_CLOEXEC`, so `exec()` releases the listening port and the backend devices
+- `load_model()` sees the restored state, skips loading and starts the queue in sleeping state; the model is then loaded upon the first request
+
+The env var is limited to 128 kB by `exec()` on Linux (`MAX_ARG_STRLEN`). If the state does not fit, the restart is skipped and the server stays in a normal sleeping state.
+
 ### Notable Related PRs
 
 - Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443

tools/server/README.md

@@ -196,11 +196,11 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) |
 | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) |
 | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
-| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
+| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
 | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/>  'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit<br/>  'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit<br/>  'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) |
 | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) |
 | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) |
-| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
+| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
 | `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)<br/>(env: LLAMA_ARG_UI) |
 | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)<br/>(env: LLAMA_ARG_EMBEDDINGS) |
 | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)<br/>(env: LLAMA_ARG_RERANKING) |
@@ -237,6 +237,7 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `-sps, --slot-prompt-similarity SIMILARITY` | how much the prompt of a request must match the prompt of a slot in order to use that slot (default: 0.10, 0.0 = disabled) |
 | `--lora-init-without-apply` | load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: disabled) |
 | `--sleep-idle-seconds SECONDS` | number of seconds of idleness after which the server will sleep (default: -1; -1 = disabled) |
+| `--sleep-mode MODE` | sleep behavior:<br/>- 'free' frees context and model memory<br/>- 'rst' restarts the whole process, may help reset memory to zero on certain backend (only support posix env)<br/>(default: free) |
 | `--log-prompts-dir PATH` | Log prompts to directory (auto-created if not present; only used for debugging, default: disabled) |
 | `--spec-draft-hf, -hfd, -hfrd, --hf-repo-draft <user>/<model>[:quant]` | Same as --hf-repo, but for the draft model (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_HF_REPO) |
 | `--spec-draft-threads, -td, --threads-draft N` | number of threads to use during generation (default: same as --threads) |
@@ -2073,6 +2074,8 @@ Note that the following endpoints are exempt from being considered as incoming t
 - `GET /models`
 - `GET /metrics`
 
+Some backends keep memory allocated even after the model is unloaded, for example a CUDA context stays on the GPU. To also release that memory, use `--sleep-mode rst`, which restarts the server process upon sleeping. The process keeps the same PID and port, and the responses of the endpoints listed above are preserved across the restart. This mode is not supported on Windows.
+
 ## More examples
 
 ### Interactive mode

tools/server/server-common.cpp

@@ -16,6 +16,18 @@
 #include <cstring>
 #include <type_traits>
 
+#if !defined(_WIN32)
+#include <unistd.h>
+#include <limits.h>
+#include <cerrno>
+#include <fcntl.h>
+#include <sys/resource.h>
+#endif
+
+#if defined(__APPLE__)
+#include <mach-o/dyld.h>
+#endif
+
 json format_error_response(const std::string & message, const enum error_type type) {
     std::string type_str;
     int code = 500;
@@ -87,6 +99,64 @@ json server_slot_stats::to_json() const {
     return base;
 }
 
+//
+// server_metrics
+//
+
+json server_metrics::bucket::to_json() const {
+    return json {
+        {"count", count},
+        {"steps", steps},
+        {"time",  time },
+    };
+}
+
+void server_metrics::bucket::from_json(const json & data) {
+    count = data.at("count");
+    steps = data.at("steps");
+    time  = data.at("time");
+}
+
+json server_metrics::to_json() const {
+    return json {
+        {"t_start",             t_start},
+
+        {"prompt_bucket",       prompt_bucket .to_json()},
+        {"predict_bucket",      predict_bucket.to_json()},
+        {"prompt",              prompt        .to_json()},
+        {"predict",             predict       .to_json()},
+
+        {"n_prompt_cached",     n_prompt_cached},
+        {"n_tokens_max",        n_tokens_max},
+        {"n_decode",            n_decode},
+        {"n_busy_slots",        n_busy_slots},
+
+        {"n_draft_tokens",      n_draft_tokens},
+        {"n_draft_accepted",    n_draft_accepted},
+        {"n_draft_verif_steps", n_draft_verif_steps},
+        {"n_accepted_per_pos",  n_accepted_per_pos},
+    };
+}
+
+void server_metrics::from_json(const json & data) {
+    t_start = data.at("t_start");
+
+    prompt_bucket .from_json(data.at("prompt_bucket"));
+    predict_bucket.from_json(data.at("predict_bucket"));
+    prompt        .from_json(data.at("prompt"));
+    predict       .from_json(data.at("predict"));
+
+    n_prompt_cached = data.at("n_prompt_cached");
+    n_tokens_max    = data.at("n_tokens_max");
+    n_decode        = data.at("n_decode");
+    n_busy_slots    = data.at("n_busy_slots");
+
+    n_draft_tokens      = data.at("n_draft_tokens");
+    n_draft_accepted    = data.at("n_draft_accepted");
+    n_draft_verif_steps = data.at("n_draft_verif_steps");
+    n_accepted_per_pos  = data.at("n_accepted_per_pos").get<std::vector<uint64_t>>();
+}
+
 //
 // random string / id
 //
@@ -1819,3 +1889,140 @@ server_tokens format_prompt_rerank(
 
     return result;
 }
+
+
+//
+// server_sleep_rst
+//
+
+#if !defined(_WIN32)
+static std::string server_proc_exe_path(char ** argv) {
+    char buf[PATH_MAX];
+#if defined(__linux__)
+    const ssize_t len = readlink("/proc/self/exe", buf, sizeof(buf) - 1);
+    if (len > 0) {
+        buf[len] = '\0';
+        return buf;
+    }
+#elif defined(__APPLE__)
+    uint32_t size = sizeof(buf);
+    if (_NSGetExecutablePath(buf, &size) == 0) {
+        return buf;
+    }
+#endif
+    return argv[0];
+}
+
+// exec() keeps the file descriptors open, so mark them all to be closed instead
+// this releases the listening port and the backend devices, and makes child processes see EOF
+static void server_proc_close_fds_on_exec() {
+    int n_fd = 4096;
+
+    struct rlimit lim;
+    if (getrlimit(RLIMIT_NOFILE, &lim) == 0 && lim.rlim_cur != RLIM_INFINITY) {
+        n_fd = std::min<int>(lim.rlim_cur, 65536);
+    }
+
+    // skip stdin/stdout/stderr, they are used to communicate with the router
+    for (int fd = 3; fd < n_fd; fd++) {
+        const int flags = fcntl(fd, F_GETFD);
+        if (flags != -1) {
+            fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
+        }
+    }
+}
+#endif
+
+static void server_proc_restart(char ** argv, const char * env_name, const std::string & env_value) {
+#if defined(_WIN32) || defined(__EMSCRIPTEN__)
+    GGML_UNUSED(argv);
+    GGML_UNUSED(env_name);
+    GGML_UNUSED(env_value);
+    SRV_ERR("%s", "restarting the process is not supported on this platform\n");
+#else
+    GGML_ASSERT(argv != nullptr);
+
+    // exec() rejects an env var larger than MAX_ARG_STRLEN (128 kB on linux)
+    if (env_value.size() > 64*1024) {
+        SRV_ERR("cannot restart the process, '%s' is too large (%zu bytes)\n", env_name, env_value.size());
+        return;
+    }
+
+    common_set_env(env_name, env_value);
+
+    const std::string exe = server_proc_exe_path(argv);
+    SRV_INF("restarting the process, exe = '%s'\n", exe.c_str());
+
+    server_proc_close_fds_on_exec();
+
+    // the log worker thread does not survive exec(), flush it while we still can
+    common_log_pause(common_log_main());
+    fflush(stdout);
+    fflush(stderr);
+
+    execv(exe.c_str(), argv);
+
+    // exec() only returns on error, the server can no longer serve requests at this point
+    GGML_ABORT("execv() failed: %s", strerror(errno));
+#endif
+}
+
+static const char * SLEEP_STATE_ENV = "LLAMA_SERVER_SLEEP_STATE";
+
+void server_sleep_rst::init(int argc, char ** argv) {
+    GGML_ASSERT(argv == nullptr || argc > 0);
+
+    this->argv = argv;
+
+    const std::string state = common_get_env(SLEEP_STATE_ENV);
+    if (state.empty()) {
+        return;
+    }
+
+    // note: the env var is kept, is_boot_to_sleep() reads it during the whole process lifetime
+    try {
+        boot_state = json::parse(state);
+    } catch (const std::exception & e) {
+        SRV_ERR("failed to read the state left by the previous process: %s\n", e.what());
+        common_set_env(SLEEP_STATE_ENV, ""); // unusable, boot normally instead
+    }
+}
+
+void server_sleep_rst::enable(common_params & params) {
+    // the state left by the previous process is unusable without the restart, drop it
+    auto disable = [this]() {
+        boot_state = json();
+        common_set_env(SLEEP_STATE_ENV, "");
+    };
+
+    if (params.sleep_mode != COMMON_SLEEP_MODE_RST) {
+        disable();
+        return;
+    }
+
+    if (argv == nullptr) {
+        // exec() can only restart a standalone process
+        SRV_WRN("%s", "--sleep-mode rst is not supported in this mode, using --sleep-mode free\n");
+        params.sleep_mode = COMMON_SLEEP_MODE_FREE;
+        disable();
+        return;
+    }
+
+    if (params.sleep_idle_seconds < 0) {
+        SRV_WRN("%s", "--sleep-mode has no effect without --sleep-idle-seconds\n");
+    }
+
+    enabled = true;
+}
+
+bool server_sleep_rst::is_boot_to_sleep() {
+    return !common_get_env(SLEEP_STATE_ENV).empty();
+}
+
+void server_sleep_rst::restart() const {
+    if (!enabled) {
+        return;
+    }
+
+    server_proc_restart(argv, SLEEP_STATE_ENV, safe_json_to_str(state_provider ? state_provider() : json()));
+}

tools/server/server-common.h

@@ -450,6 +450,9 @@ struct server_metrics {
             steps += n_steps;
             time  += t_us;
         }
+
+        json to_json() const;
+        void from_json(const json & data);
     };
 
     // these are reset by reset_bucket(), only the rate is read from them
@@ -490,6 +493,10 @@ struct server_metrics {
     void add_prompt_cached(uint64_t n_tokens) {
         n_prompt_cached += n_tokens;
     }
+
+    // used to keep the metrics across a process restart, see --sleep-mode rst
+    json to_json() const;
+    void from_json(const json & data);
 };
 
 //
@@ -604,3 +611,39 @@ struct server_pipe {
         return true;
     }
 };
+
+//
+// server_sleep_rst
+// this allow --sleep-mode rst to reset the whole process, but still preserve metrics and props data
+// see README-dev.md for more info
+//
+
+struct server_sleep_rst {
+    // true if the process was restarted by a previous instance
+    // in this case, the model is only loaded upon the first request
+    static bool is_boot_to_sleep();
+
+    // remember argv and read the state left by the previous process
+    // must be called once at startup, before spawning any thread or child process
+    void init(int argc, char ** argv);
+
+    // enable the restart upon sleeping, warns and falls back to --sleep-mode free if not possible
+    void enable(common_params & params);
+
+    // state left by the previous process, only valid if is_boot_to_sleep()
+    const json & get_boot_state() const { return boot_state; }
+
+    // set the state to be preserved across the restart
+    void set_state_provider(std::function<json()> provider) { state_provider = std::move(provider); }
+
+    // restart the process, does nothing if not enabled, does not return on success
+    void restart() const;
+
+private:
+    bool    enabled = false;
+    char ** argv    = nullptr;
+
+    json boot_state;
+
+    std::function<json()> state_provider;
+};

tools/server/server-context.cpp

@@ -798,6 +798,15 @@ public:
     mtmd_context * mctx = nullptr;
     const llama_vocab * vocab = nullptr;
 
+    server_sleep_rst sleep_rst;
+
+    server_metrics metrics;
+
+    // called each time the model is loaded, used by server_routes to refresh its metadata
+    void on_model_loaded(std::function<void()> callback) {
+        callback_model_loaded = std::move(callback);
+    }
+
     server_queue    queue_tasks;
     server_response queue_results;
 
@@ -818,14 +827,6 @@ public:
         }
     }
 
-    server_metrics get_metrics() const {
-        return metrics;
-    }
-
-    void reset_metrics_bucket() {
-        metrics.reset_bucket();
-    }
-
 private:
     // note: accessing these fields outside of this class is not thread-safe
     // use server_context methods instead
@@ -866,8 +867,6 @@ private:
 
     std::unique_ptr<server_prompt_cache> prompt_cache;
 
-    server_metrics metrics;
-
     // queued prompt stats - llama_decode() is async, so the timing is only valid after a sync
     // note: kept out of server_metrics, which is copied as-is into the task result
     int64_t  t_decode_start  = 0; // start of the last submitted decode
@@ -885,6 +884,13 @@ private:
 
     bool sleeping = false;
 
+    // set once init() has run, which requires a loaded model
+    bool initialized = false;
+
+    bool queue_initialized = false;
+
+    std::function<void()> callback_model_loaded;
+
     int64_t t_last_load_progress_ms = 0;
 
     void destroy() {
@@ -912,11 +918,15 @@ private:
             }
             SRV_INF("%s", "server is entering sleeping state\n");
             destroy();
-        } else {
-            SRV_INF("%s", "server is exiting sleeping state\n");
-            if (!load_model(params_base)) {
-                GGML_ABORT("failed to reload model after sleeping");
-            }
+            sleeping = new_state;
+            // everything is released, the process can now restart itself
+            sleep_rst.restart();
+            return;
+        }
+
+        SRV_INF("%s", "server is exiting sleeping state\n");
+        if (!load_model(params_base)) {
+            GGML_ABORT("failed to reload model after sleeping");
         }
         sleeping = new_state;
     }
@@ -956,6 +966,18 @@ private:
     // load the model and initialize llama_context
     // this may also be called to resume from sleeping state
     bool load_model(common_params & params) {
+        if (!initialized && !sleeping) {
+            sleep_rst.enable(params);
+
+            if (sleep_rst.is_boot_to_sleep()) {
+                init_sleeping(params);
+                return true;
+            }
+        }
+
+        llama_backend_init();
+        llama_numa_init(params.numa);
+
         load_progress_data load_progress_text  (this, "text_model");
         load_progress_data load_progress_mmproj(this, "mmproj_model");
         load_progress_data load_progress_spec  (this, "spec_model");
@@ -1349,25 +1371,34 @@ private:
         // propagate new defaults back to caller
         params = params_base;
 
-        if (!is_resume) {
-            return init();
+        // a process restarted into sleeping state loads the model here for the first time
+        if (!initialized) {
+            initialized = true;
+            if (!init()) {
+                return false;
+            }
         }
 
-        if (callback_state) {
+        SRV_INF("%s", "model loaded\n");
+
+        if (callback_model_loaded) {
+            callback_model_loaded();
+        }
+
+        if (is_resume && callback_state) {
             callback_state(SERVER_STATE_READY, {});
         }
 
         return true;
     }
 
-    // unlike load_model(), this is only called once during initialization
-    bool init() {
-        GGML_ASSERT(ctx_tgt   != nullptr);
-        GGML_ASSERT(model_tgt != nullptr);
+    // wiring up server queues, must be done once before start_loop()
+    void init_queue() {
+        if (queue_initialized) {
+            return; // already done by init_sleeping()
+        }
+        queue_initialized = true;
 
-        GGML_ASSERT(!sleeping);
-
-        // wiring up server queues
         queue_tasks.on_new_task([this](server_task && task, bool is_yielding) {
             return process_single_task(std::move(task), is_yielding);
         });
@@ -1379,6 +1410,28 @@ private:
         });
 
         metrics.init();
+    }
+
+    // enter sleeping state without a model, the model is loaded upon leaving that state
+    void init_sleeping(common_params & params) {
+        GGML_ASSERT(!initialized);
+
+        params_base = params;
+
+        init_queue();
+
+        sleeping = true;
+        queue_tasks.init_sleeping();
+
+        SRV_INF("%s", "restarted in sleeping state, the model will be loaded upon the first request\n");
+    }
+
+    // unlike load_model(), this is only called once during initialization
+    bool init() {
+        GGML_ASSERT(ctx_tgt   != nullptr);
+        GGML_ASSERT(model_tgt != nullptr);
+
+        init_queue();
 
         if (params_base.cache_idle_slots) {
             if (params_base.cache_ram_mib == 0) {
@@ -4074,6 +4127,10 @@ private:
 server_context::server_context() : impl(new server_context_impl()) {}
 server_context::~server_context() = default;
 
+void server_context::init(int argc, char ** argv) {
+    impl->sleep_rst.init(argc, argv);
+}
+
 bool server_context::load_model(common_params & params) {
     return impl->load_model(params);
 }
@@ -4454,6 +4511,21 @@ server_routes::server_routes(const common_params & params, server_context & ctx_
     queue_tasks.on_sleeping_state([this](bool is_sleeping) {
         update_cached_responses(is_sleeping);
     });
+
+    // meta is only available once the model is loaded, which may happen upon leaving sleeping state
+    this->ctx_server.on_model_loaded([this, &ctx_server]() {
+        update_meta(ctx_server);
+    });
+
+    // set the hook to allow sleep_rst to capture the state BEFORE resetting the process
+    this->ctx_server.sleep_rst.set_state_provider([this]() {
+        return cache_to_json();
+    });
+
+    // reverse of above: if we just booted AFTER sleep_rst reset the process, we restore it
+    if (this->ctx_server.sleep_rst.is_boot_to_sleep()) {
+        cache_from_json(this->ctx_server.sleep_rst.get_boot_state());
+    }
 }
 
 static json get_res_model_info(const server_context_meta & meta) {
@@ -5459,7 +5531,7 @@ void server_routes::update_cached_responses(bool is_sleeping) {
     if (is_sleeping) {
         cached_models  = get_res_models(*meta);
         cached_props   = get_res_props(*meta, params, true);
-        cached_metrics = ctx_server.get_metrics();
+        cached_metrics = ctx_server.metrics;
 
         should_reset_buckets = false;
 
@@ -5467,8 +5539,37 @@ void server_routes::update_cached_responses(bool is_sleeping) {
 
     } else if (should_reset_buckets) {
         // a scrape during sleep already reported these buckets
-        ctx_server.reset_metrics_bucket();
+        ctx_server.metrics.reset_bucket();
 
         should_reset_buckets = false;
     }
 }
+
+json server_routes::cache_to_json() {
+    std::unique_lock<std::mutex> lock(mutex_cache);
+
+    return json {
+        {"models",  cached_models},
+        {"props",   cached_props},
+        {"metrics", cached_metrics.to_json()},
+    };
+}
+
+bool server_routes::cache_from_json(const json & data) {
+    std::unique_lock<std::mutex> lock(mutex_cache);
+
+    try {
+        cached_models = data.at("models");
+        cached_props  = data.at("props");
+        cached_metrics.from_json(data.at("metrics"));
+    } catch (const std::exception & e) {
+        SRV_ERR("failed to restore cached responses: %s\n", e.what());
+        return false;
+    }
+
+    // keep counting from where the previous process stopped
+    ctx_server.metrics = cached_metrics;
+
+    return true;
+}
+

tools/server/server-context.h

@@ -87,8 +87,14 @@ struct server_context {
     server_context();
     ~server_context();
 
+    // remember the command line, needed to restart the process, see --sleep-mode rst
+    // must be called once at startup, before spawning any thread or child process
+    void init(int argc, char ** argv);
+
     // load the model and initialize llama_context
     // returns true on success
+    // note: if the process was restarted into sleeping state, no model is loaded and it
+    //       returns true right away, the model is then loaded upon the first request
     bool load_model(common_params & params);
 
     // this function will block main thread until termination
@@ -158,6 +164,11 @@ struct server_routes {
     // to be used in router mode
     json get_model_info() const;
 
+    // save / restore the cached responses across a process restart, see --sleep-mode rst
+    // only valid while sleeping, as the cache is only updated upon entering that state
+    json cache_to_json();
+    bool cache_from_json(const json & data);
+
 private:
     std::unique_ptr<server_res_generator> handle_completions_impl(
             const server_http_req & req,
@@ -191,3 +202,4 @@ private:
     // call right before sleep to update the cached responses
     void update_cached_responses(bool is_sleeping);
 };
+

tools/server/server-queue.cpp

@@ -286,6 +286,28 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
     worker.yielding = false;
     worker.thread = std::thread([this]() { worker_loop(); });
 
+    // the process may start already sleeping, see init_sleeping()
+    {
+        std::unique_lock<std::mutex> lock(mutex_tasks);
+        if (sleeping) {
+            QUE_INF("%s", "starting in sleeping state\n");
+            condition_tasks.wait(lock, [&]{
+                return (!running || req_stop_sleeping);
+            });
+            if (running) {
+                QUE_INF("%s", "exiting sleeping state\n");
+                req_stop_sleeping = false;
+                // Call order cb{N} -> cb1 -> cb0
+                for (size_t i = callback_sleeping_state.size(); i > 0; i--) {
+                    callback_sleeping_state[i - 1](false);
+                }
+                sleeping = false;
+                condition_tasks.notify_all(); // notify wait_until_no_sleep()
+            }
+            time_last_task = ggml_time_ms();
+        }
+    }
+
     constexpr auto max_wait_time = std::chrono::seconds(1);
     auto should_sleep = [&]() -> bool {
         // caller must hold mutex_tasks

tools/server/server-queue.h

@@ -74,6 +74,13 @@ public:
         return sleeping;
     }
 
+    // enter sleeping state before start_loop(), for a process restarted into that state
+    // note: callback_sleeping_state(true) is not called, the state is already known
+    void init_sleeping() {
+        std::unique_lock<std::mutex> lock(mutex_tasks);
+        sleeping = true;
+    }
+
     // end the start_loop routine
     void terminate();
 

tools/server/server.cpp

@@ -106,9 +106,6 @@ int llama_server(int argc, char ** argv) {
         return 1;
     }
 
-    llama_backend_init();
-    llama_numa_init(params.numa);
-
     return llama_server(params, argc, argv);
 }
 
@@ -135,8 +132,10 @@ int llama_server(common_params & params, int argc, char ** argv) {
     const bool is_router_server = params.model.path.empty()
                                && params.model.hf_repo.empty();
 
-    // skip device enumeration so the CUDA primary context stays uncreated
-    common_params_print_info(params, !is_router_server);
+    // may skip device enumeration so the CUDA primary context stays uncreated
+    if (!server_sleep_rst::is_boot_to_sleep()) {
+        common_params_print_info(params, !is_router_server);
+    }
 
     if (!is_router_server) {
         // validate batch size for embeddings
@@ -167,6 +166,7 @@ int llama_server(common_params & params, int argc, char ** argv) {
 
     // struct that contains llama context and inference
     server_context ctx_server;
+    ctx_server.init(argc, argv);
 
     server_http_context ctx_http;
     if (!ctx_http.init(params)) {
@@ -458,11 +458,8 @@ int llama_server(common_params & params, int argc, char ** argv) {
             return 1;
         }
 
-        routes.update_meta(ctx_server);
         ctx_http.is_ready.store(true);
 
-        SRV_INF("%s", "model loaded\n");
-
         shutdown_handler = [&](int) {
             mcp_mgr.shutdown();
             // this will unblock start_loop()
@@ -513,7 +510,10 @@ int llama_server(common_params & params, int argc, char ** argv) {
         std::thread monitor_thread;
         if (child.is_child()) {
             monitor_thread = child.setup(shutdown_handler);
-            child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
+            // if no model is loaded, the process restarted into sleeping state and the router knows it
+            if (ctx_server.get_llama_context() != nullptr) {
+                child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
+            }
         }
 
         // this call blocks the main thread until queue_tasks.terminate() is called