diff --git a/tools/ui/src/lib/constants/backend.constants.ts b/tools/ui/src/lib/constants/backend.constants.ts index 7008333c79..5145afbdee 100644 --- a/tools/ui/src/lib/constants/backend.constants.ts +++ b/tools/ui/src/lib/constants/backend.constants.ts @@ -23,6 +23,7 @@ const LLAMA_CPP_CAPABILITIES: BackendCapabilities = { corsProxy: true, loadUnload: true, props: true, + resumableStreams: true, router: true, slots: true, statusFeed: true, @@ -33,6 +34,7 @@ const COMPATIBLE_CAPABILITIES: BackendCapabilities = { corsProxy: false, loadUnload: false, props: false, + resumableStreams: false, router: false, slots: false, statusFeed: false, diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 31cefe3c9a..2801a902d1 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -43,12 +43,47 @@ import type { ApiStreamSession } from '$lib/types/api'; import { isAbortError } from '$lib/utils/abort'; -import { apiChatUrl, apiUrl } from '$lib/utils/api-base'; +import { apiChatUrl, apiUrl, getBackend } from '$lib/utils/api-base'; import { ApiError } from '$lib/utils/api-fetch'; import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { streamIdentity } from '$lib/utils/stream-identity'; +/** + * llama.cpp-only chat request fields. Strict OpenAI-compatible endpoints + * reject unknown parameters, so they are dropped for those backends. + */ +const COMPAT_ONLY_OMIT_REQUEST_FIELDS = [ + 'add_generation_prompt', + 'backend_sampling', + 'cache_prompt', + 'chat_template_kwargs', + 'continue_final_message', + 'dry_allowed_length', + 'dry_base', + 'dry_multiplier', + 'dry_penalty_last_n', + 'dynatemp_exponent', + 'dynatemp_range', + 'id_slot', + 'min_p', + 'n_keep', + 'n_predict', + 'reasoning_control', + 'reasoning_format', + 'repeat_last_n', + 'repeat_penalty', + 'return_progress', + 'samplers', + 'sse_ping_interval', + 'thinking_budget_tokens', + 'timings_per_token', + 'top_k', + 'typ_p', + 'xtc_probability', + 'xtc_threshold' +]; + interface ResumableStreamState { bytesReceived: number; updatedAt: number; @@ -111,6 +146,8 @@ export class ChatService { static async cancelServerStream(conversationId: string, model?: string | null): Promise { if (!conversationId) return; + if (!serverStore.capabilities.resumableStreams) return; + try { const id = streamIdentity(conversationId, model); @@ -344,6 +381,10 @@ export class ChatService { * caller can pipe it through the SSE parser like a fresh stream. */ static async fetchStreamReplay(streamId: string): Promise { + if (!serverStore.capabilities.resumableStreams) { + return new Response(null, { status: 501, statusText: 'Not Implemented' }); + } + const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { headers: getAuthHeaders() }); @@ -786,6 +827,8 @@ export class ChatService { * conv::model identity when a model was bound at POST time. */ static async lookupStreamSessions(conversationIds: string[]): Promise { + if (!serverStore.capabilities.resumableStreams) return []; + const resp = await fetch(apiUrl(API_STREAM.LOOKUP), { body: JSON.stringify({ conversation_ids: conversationIds }), headers: getJsonHeaders(), @@ -848,6 +891,9 @@ export class ChatService { excludeReasoning?: boolean, signal?: AbortSignal ): Promise { + // pre-encode warms the llama.cpp KV cache and posts llama.cpp-only fields + if (!serverStore.capabilities.props) return; + const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi(messages); const requestBody: Record = { @@ -892,6 +938,8 @@ export class ChatService { static async probeResumeStatus(streamId: string): Promise { if (!streamId) return 0; + if (!serverStore.capabilities.resumableStreams) return 0; + const ac = new AbortController(); try { @@ -915,6 +963,8 @@ export class ChatService { ): Promise { if (!conversationId) return null; + if (!serverStore.capabilities.resumableStreams) return null; + const state = ChatService.getStreamState(conversationId); const from = state?.bytesReceived ?? 0; const id = streamIdentity(conversationId, model); @@ -1235,6 +1285,8 @@ export class ChatService { ChatService.saveStreamState(conversationId, 0, options.model ?? null); } + ChatService.stripBackendSpecificFields(requestBody); + const response = await fetch(apiChatUrl(), { body: JSON.stringify(requestBody), headers, @@ -1630,6 +1682,28 @@ export class ChatService { * Strips legacy inline reasoning content tags from message content. * Handles both plain string content and multipart content arrays. */ + private static stripBackendSpecificFields(body: ApiChatCompletionRequest): void { + const backend = getBackend(); + + if (!backend || backend.protocol === 'llama.cpp') return; + + const record = body as unknown as Record; + + for (const field of COMPAT_ONLY_OMIT_REQUEST_FIELDS) { + delete record[field]; + } + + // compatible endpoints reject the reasoning_content message extension + for (const message of body.messages) { + delete (message as unknown as Record).reasoning_content; + } + + // -1 is llama.cpp's "no limit" sentinel; compatible endpoints reject it + if (typeof body.max_tokens === 'number' && body.max_tokens <= 0) { + delete record.max_tokens; + } + } + private static stripReasoningContent( content: string | ApiChatMessageContentPart[] ): string | ApiChatMessageContentPart[] { diff --git a/tools/ui/src/lib/stores/models/props.svelte.ts b/tools/ui/src/lib/stores/models/props.svelte.ts index 9d2d817acb..e388f46ddc 100644 --- a/tools/ui/src/lib/stores/models/props.svelte.ts +++ b/tools/ui/src/lib/stores/models/props.svelte.ts @@ -152,6 +152,9 @@ export class ModelPropsManager { * @returns Props data or null if fetch failed or model not loaded */ async fetchModelProps(modelId: string): Promise { + // /props only exists on llama.cpp servers + if (!serverStore.capabilities.props) return null; + const cached = this.cache.get(modelId); if (cached) return cached; diff --git a/tools/ui/src/lib/types/backend.d.ts b/tools/ui/src/lib/types/backend.d.ts index 0b7d709c54..43b071b4a4 100644 --- a/tools/ui/src/lib/types/backend.d.ts +++ b/tools/ui/src/lib/types/backend.d.ts @@ -22,6 +22,8 @@ export interface BackendCapabilities { loadUnload: boolean; /** The /props endpoint with server role and generation defaults. */ props: boolean; + /** Resumable stream sessions (/v1/stream, /v1/streams/lookup). */ + resumableStreams: boolean; /** Multi-model router mode. */ router: boolean; /** The /slots introspection endpoint. */ diff --git a/tools/ui/src/lib/utils/api-key-validation.ts b/tools/ui/src/lib/utils/api-key-validation.ts index b747936e42..22f84c604c 100644 --- a/tools/ui/src/lib/utils/api-key-validation.ts +++ b/tools/ui/src/lib/utils/api-key-validation.ts @@ -3,7 +3,8 @@ import { browser } from '$app/environment'; import { HEADERS } from '$lib/constants'; import { MimeTypeApplication } from '$lib/enums'; import { settingsStore } from '$lib/stores/settings/index.svelte'; -import { apiUrl } from '$lib/utils/api-base'; +import { apiUrl, getBackend } from '$lib/utils/api-base'; +import { getBackendCapabilities } from '$lib/utils/backend'; /** * Validates API key by making a request to the server props endpoint @@ -14,6 +15,13 @@ export async function validateApiKey(fetch: typeof globalThis.fetch): Promise