(sip) allow inject messge to generation

tools/server/README.md

@@ -1443,7 +1443,11 @@ Acts on an in-flight completion identified by its `id` (the `id` field streamed
 
 `id`: (Required) The chat completion id to act on. A completion that has already finished matches nothing and the call is a no-op.
 
-`action`: (Required) The control action to perform. Currently the only supported value is `reasoning_end`, which forces the end of the current reasoning block so the model moves on to the final answer. Requires `reasoning_control: true` on the original completion request.
+`action`: (Required) The control action to perform. Supported values:
+- `reasoning_end`: forces the end of the current reasoning block so the model moves on to the final answer. Requires `reasoning_control: true` on the original completion request.
+- `inject`: appends the given `text` to the generated output as if the model produced it, then generation continues from there. Useful to steer the model mid reasoning by injecting a thought. Fails if the completion is still processing the prompt, or if it is constrained by a grammar / JSON schema. Injected tokens are streamed back like sampled tokens and count towards `max_tokens`.
+
+`text`: (Required for `inject`) The text to inject.
 
 `model`: (Required in router mode) The model name, used to route the request to the right instance. Ignored in single model mode.
 

tools/server/server-context.cpp

@@ -305,6 +305,9 @@ struct server_slot {
 
     llama_token sampled; // in speculative mode, this is the last accepted token
 
+    // tokens queued by the control endpoint "inject" action, consumed instead of sampling
+    llama_tokens inject_tokens;
+
     // for TTS models, this is the embd generated from prev step, decode this to generate next hidden state
     // corresponding to one token position (size = n_embd)
     std::vector<float> inp_embd;
@@ -342,6 +345,7 @@ struct server_slot {
         }
         generated_tokens.clear();
         generated_token_probs.clear();
+        inject_tokens.clear();
         json_schema = json();
 
         task_prev = std::move(task);
@@ -443,6 +447,11 @@ struct server_slot {
             return 0;
         }
 
+        // do not draft while injected tokens are pending
+        if (!inject_tokens.empty()) {
+            return 0;
+        }
+
         // determine the max draft that fits the current slot state
         // note: slot.prompt is not yet expanded with the `id` token sampled above
         //       also, need to leave space for 1 extra token to allow context shifts
@@ -2404,6 +2413,24 @@ private:
                         // act on the live slot mid generation, never defer
                         common_sampler_reasoning_budget_force(slot->smpl.get());
                         res->success = true;
+                    } else if (task.params.control_action == "inject") {
+                        if (slot->state != SLOT_STATE_GENERATING) {
+                            res->success = false;
+                            res->message = "completion is still processing the prompt";
+                        } else if (!slot->task->params.sampling.grammar.empty() && !slot->task->params.sampling.grammar_lazy) {
+                            // a lazy grammar (tool calls with auto choice) is fine as long as the injected text does not trigger it
+                            res->success = false;
+                            res->message = "cannot inject into a completion constrained by a grammar";
+                        } else {
+                            const llama_tokens tokens = common_tokenize(vocab, task.params.control_text, false, true);
+                            if (tokens.empty()) {
+                                res->success = false;
+                                res->message = "text produced no tokens";
+                            } else {
+                                slot->inject_tokens.insert(slot->inject_tokens.end(), tokens.begin(), tokens.end());
+                                res->success = true;
+                            }
+                        }
                     } else {
                         res->success = false;
                         res->message = "unknown control action";
@@ -3757,14 +3784,31 @@ private:
             const int tok_idx = slot.i_batch - off;
 
             llama_token id;
-            {
+            const bool injected = !slot.inject_tokens.empty();
+            if (injected) {
+                // consume an injected token instead of sampling
+                id = slot.inject_tokens.front();
+                slot.inject_tokens.erase(slot.inject_tokens.begin());
+            } else {
                 scoped_timer timer(t_sampl, n_sampl);
                 id = common_sampler_sample(slot.smpl.get(), slot.ctx_tgt, tok_idx);
             }
 
             slot.i_batch = -1;
 
-            common_sampler_accept(slot.smpl.get(), id, true);
+            if (injected) {
+                // an injected token can violate a lazy grammar; fail only this slot instead of aborting all slots
+                try {
+                    common_sampler_accept(slot.smpl.get(), id, true);
+                } catch (const std::exception & e) {
+                    SLT_ERR(slot, "injected token rejected by grammar: %s\n", e.what());
+                    send_error(slot, "injected text conflicts with the active grammar", ERROR_TYPE_INVALID_REQUEST);
+                    slot.release();
+                    return;
+                }
+            } else {
+                common_sampler_accept(slot.smpl.get(), id, true);
+            }
 
             // here we have synchronized the llama_context (due to the sampling above), so we can do time measurement
             const int64_t t_now = ggml_time_us();
@@ -4756,14 +4800,19 @@ void server_routes::init_routes() {
 
         const std::string cmpl_id = json_value(body, "id", std::string());
         const std::string action  = json_value(body, "action", std::string());
+        const std::string text    = json_value(body, "text", std::string());
         if (cmpl_id.empty()) {
             res->error(format_error_response("missing completion id", ERROR_TYPE_INVALID_REQUEST));
             return res;
         }
-        if (action != "reasoning_end") {
+        if (action != "reasoning_end" && action != "inject") {
             res->error(format_error_response("unknown control action", ERROR_TYPE_INVALID_REQUEST));
             return res;
         }
+        if (action == "inject" && text.empty()) {
+            res->error(format_error_response("missing text for inject action", ERROR_TYPE_INVALID_REQUEST));
+            return res;
+        }
 
         auto & rd = res->rd;
         {
@@ -4771,6 +4820,7 @@ void server_routes::init_routes() {
             task.id              = rd.get_new_id();
             task.params.control_cmpl_id = cmpl_id;
             task.params.control_action  = action;
+            task.params.control_text    = text;
             rd.post_task(std::move(task));
         }
 

tools/server/server-task.h

@@ -87,6 +87,7 @@ struct task_params {
     // realtime control (SERVER_TASK_TYPE_CONTROL)
     std::string        control_action;
     std::string        control_cmpl_id;
+    std::string        control_text; // for "inject" action
 
     // per-request parameters for chat parsing
     common_chat_parser_params chat_parser_params;

tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte

@@ -129,9 +129,8 @@
 
 		handleSendLikeScroll();
 
-		await chatStore.sendMessage(message, result?.extras);
-
-		return true;
+		// false means a mid-generation inject failed; the form restores the input
+		return await chatStore.sendMessage(message, result?.extras);
 	}
 
 	let lastScrolledConversationId: string | null = null;

tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte

@@ -84,6 +84,7 @@
 
 		if (!chatFormRef?.checkModelSelected()) return;
 
+		const originalMessage = message;
 		const messageToSend = message.trim();
 		const filesToSend = [...uploadedFiles];
 
@@ -96,7 +97,7 @@
 		const success = await onSend?.(messageToSend, filesToSend);
 
 		if (!success) {
-			message = messageToSend;
+			message = originalMessage;
 			uploadedFiles = filesToSend;
 		}
 	}

tools/ui/src/lib/constants/control-actions.constants.ts

@@ -1,5 +1,6 @@
 // actions accepted by the realtime inference control endpoint (API_CHAT.CONTROL)
 // kept separate from the endpoint paths since these are protocol level verbs
 export const CONTROL_ACTION = {
-	END_REASONING: 'reasoning_end'
+	END_REASONING: 'reasoning_end',
+	INJECT: 'inject'
 } as const;

tools/ui/src/lib/constants/settings-keys.constants.ts

@@ -6,6 +6,7 @@
  */
 export const SETTINGS_KEYS = {
 	AGENTIC_MAX_TURNS: 'agenticMaxTurns',
+	ALLOW_INJECT_USER_MESSAGE_MID_GENERATION: 'allowInjectUserMessageMidGeneration',
 	ALWAYS_SHOW_SIDEBAR_ON_DESKTOP: 'alwaysShowSidebarOnDesktop',
 	ALWAYS_SHOW_TOOL_CALL_CONTENT: 'alwaysShowToolCallContent',
 	API_KEY: 'apiKey',
@@ -28,6 +29,7 @@ export const SETTINGS_KEYS = {
 	EXCLUDE_REASONING_FROM_CONTEXT: 'excludeReasoningFromContext',
 	FREQUENCY_PENALTY: 'frequency_penalty',
 	FULL_HEIGHT_CODE_BLOCKS: 'fullHeightCodeBlocks',
+	INJECTION_TEMPLATE: 'injectionTemplate',
 	JS_SANDBOX_ENABLED: 'jsSandboxEnabled',
 	MAX_IMAGE_RESOLUTION: 'maxImageMPixels',
 	MAX_TOKENS: 'max_tokens',

tools/ui/src/lib/constants/settings-registry.constants.ts

@@ -136,6 +136,24 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
 				section: SETTINGS_SECTION_SLUGS.DEVELOPER,
 				type: SettingsFieldType.CHECKBOX
 			},
+			{
+				defaultValue: false,
+				help: 'Send a message typed during a streaming response straight into the model output via the control endpoint, instead of queueing it until the response finishes. Text only; messages with attachments are still queued. The injected text appears inside the assistant response.',
+				key: SETTINGS_KEYS.ALLOW_INJECT_USER_MESSAGE_MID_GENERATION,
+				label: 'Allow inject user message mid-generation',
+				section: SETTINGS_SECTION_SLUGS.DEVELOPER,
+				type: SettingsFieldType.CHECKBOX
+			},
+			{
+				defaultValue:
+					'<system_message>\nUser wants interrupt without stopping this session. Therefore, you are seeing this as part of your reasoning process:\n{message}\n</system_message>',
+				dependsOn: SETTINGS_KEYS.ALLOW_INJECT_USER_MESSAGE_MID_GENERATION,
+				help: 'Template applied to a message injected mid-generation. {message} is replaced with the typed text.',
+				key: SETTINGS_KEYS.INJECTION_TEMPLATE,
+				label: 'Injection template',
+				section: SETTINGS_SECTION_SLUGS.DEVELOPER,
+				type: SettingsFieldType.TEXTAREA
+			},
 			{
 				defaultValue: false,
 				help: 'Strip thinking from previous messages before sending. When off, thinking is sent back via the reasoning_content field so the model sees its own chain-of-thought across turns.',

tools/ui/src/lib/services/chat.service.ts

@@ -508,6 +508,58 @@ export class ChatService {
 		}
 	}
 
+	/**
+	 * Injects text into a running completion, targeted by its chat completion id.
+	 * The server appends the tokens to the generated output as if the model
+	 * produced them and streams them back. Returns true on success.
+	 */
+	static async injectText(
+		completionId: string,
+		text: string,
+		model?: string | null
+	): Promise<boolean> {
+		if (!completionId) {
+			console.error(
+				'injectText: no completion id for the active message, cannot target the running completion'
+			);
+
+			return false;
+		}
+
+		const body: Record<string, unknown> = {
+			action: CONTROL_ACTION.INJECT,
+			id: completionId,
+			text
+		};
+
+		if (model) body.model = model;
+
+		try {
+			const res = await fetch(API_CHAT.CONTROL, {
+				body: JSON.stringify(body),
+				headers: getJsonHeaders(),
+				method: 'POST'
+			});
+			const data = await res.json().catch(() => null);
+
+			if (!res.ok || data?.success !== true) {
+				console.error('injectText: control request failed', {
+					completionId,
+					response: data,
+					status: res.status
+				});
+
+				return false;
+			}
+
+			return true;
+		} catch (error) {
+			console.error('injectText: control request threw', { completionId, error });
+
+			return false;
+		}
+	}
+
 	/**
 	 * Sends a fire-and-forget request to pre-encode the conversation in the server's KV cache.
 	 * After a response completes, this re-submits the full conversation

tools/ui/src/lib/stores/chat.svelte.ts

@@ -1187,23 +1187,37 @@ class ChatStore {
 		);
 	}
 
-	async sendMessage(content: string, extras?: DatabaseMessageExtra[]): Promise<void> {
-		if (!content.trim() && (!extras || extras.length === 0)) return;
+	/** Returns false only when a mid-generation inject failed, so the caller can restore the input. */
+	async sendMessage(content: string, extras?: DatabaseMessageExtra[]): Promise<boolean> {
+		if (!content.trim() && (!extras || extras.length === 0)) return true;
 
 		const activeConv = conversationsStore.activeConversation;
 
+		// text only for now: messages with attachments always take the steering/queue path
+		const preferDirectInject =
+			Boolean(settingsStore.config.allowInjectUserMessageMidGeneration) &&
+			(!extras || extras.length === 0);
+
 		// If agentic loop is running, inject as a steering message instead of starting a new flow
 		if (activeConv && agenticStore.isRunning(activeConv.id)) {
+			if (preferDirectInject) {
+				return await this.injectIntoActiveCompletion(content);
+			}
+
 			agenticStore.injectSteeringMessage(activeConv.id, content, extras);
 
-			return;
+			return true;
 		}
 
 		// If non-agentic streaming is active, queue as a pending message to send after completion
 		if (activeConv && this.isChatLoadingInternal(activeConv.id)) {
+			if (preferDirectInject) {
+				return await this.injectIntoActiveCompletion(content);
+			}
+
 			this.injectPendingMessage(activeConv.id, content, extras);
 
-			return;
+			return true;
 		}
 
 		// Cancel any in-flight pre-encode request
@@ -1222,7 +1236,7 @@ class ChatStore {
 
 		const currentConv = conversationsStore.activeConversation;
 
-		if (!currentConv) return;
+		if (!currentConv) return true;
 
 		this.showErrorDialog(null);
 		this.setChatLoading(currentConv.id, true);
@@ -1300,7 +1314,7 @@ class ChatStore {
 			if (isAbortError(error)) {
 				this.setChatLoading(currentConv.id, false);
 
-				return;
+				return true;
 			}
 
 			console.error('Failed to send message:', error);
@@ -1319,6 +1333,37 @@ class ChatStore {
 				type: dialogType
 			});
 		}
+
+		return true;
+	}
+
+	/**
+	 * Sends the text into the running completion of the active conversation via the
+	 * control endpoint. The injected text is streamed back as part of the assistant
+	 * response, so no message is added to the conversation.
+	 */
+	private async injectIntoActiveCompletion(content: string): Promise<boolean> {
+		const messages = conversationsStore.activeMessages;
+		const activeMessage = messages[messages.length - 1];
+
+		if (!activeMessage?.completionId) return false;
+
+		const template = String(settingsStore.config.injectionTemplate ?? '');
+
+		let rendered = content;
+
+		if (template.includes('{message}')) {
+			rendered = template.replaceAll('{message}', content);
+		} else if (template) {
+			rendered = `${template}\n${content}`;
+		}
+
+		// surround with newlines so the text does not glue to the tokens around it
+		return await ChatService.injectText(
+			activeMessage.completionId,
+			`\n${rendered}\n`,
+			activeMessage.model
+		);
 	}
 
 	private async streamChatCompletion(