diff --git a/tools/ui/src/lib/components/app/backends/BackendForm.svelte b/tools/ui/src/lib/components/app/backends/BackendForm.svelte index 641e217b50..992040f779 100644 --- a/tools/ui/src/lib/components/app/backends/BackendForm.svelte +++ b/tools/ui/src/lib/components/app/backends/BackendForm.svelte @@ -67,7 +67,7 @@ API format onChange({ protocol: value as BackendProtocol })} + onValueChange={(value) => onChange({ compat: undefined, protocol: value as BackendProtocol })} type="single" value={backend.protocol} > diff --git a/tools/ui/src/lib/components/app/backends/DialogBackendForm.svelte b/tools/ui/src/lib/components/app/backends/DialogBackendForm.svelte index fd464c6ff0..1917b945bf 100644 --- a/tools/ui/src/lib/components/app/backends/DialogBackendForm.svelte +++ b/tools/ui/src/lib/components/app/backends/DialogBackendForm.svelte @@ -66,6 +66,7 @@ ...draft, baseUrl: preset.baseUrl, chatPath: preset.chatPath, + compat: preset.compat, modelsPath: preset.modelsPath, name: preset.id === 'custom' ? '' : preset.name, protocol: preset.protocol diff --git a/tools/ui/src/lib/constants/backend.constants.ts b/tools/ui/src/lib/constants/backend.constants.ts index 5145afbdee..fc282720e4 100644 --- a/tools/ui/src/lib/constants/backend.constants.ts +++ b/tools/ui/src/lib/constants/backend.constants.ts @@ -1,4 +1,9 @@ -import type { BackendCapabilities, BackendPreset, BackendProtocol } from '$lib/types'; +import type { + BackendCapabilities, + BackendCompat, + BackendPreset, + BackendProtocol +} from '$lib/types'; /** Version sent with the Anthropic Messages API. */ export const ANTHROPIC_API_VERSION = '2023-06-01'; @@ -48,6 +53,15 @@ export const BACKEND_CAPABILITIES: Record openai: COMPATIBLE_CAPABILITIES }; +/** Default wire quirks per protocol. */ +export const BACKEND_COMPAT: Record = { + // the Messages API has no OpenAI-style token cap or usage-in-stream toggle + anthropic: { maxTokensField: 'max_tokens', supportsUsageInStreaming: false }, + // llama-server reports its own timings, so it needs no usage chunk + 'llama.cpp': { maxTokensField: 'max_tokens', supportsUsageInStreaming: false }, + openai: { maxTokensField: 'max_tokens', supportsUsageInStreaming: true } +}; + /** * Ready-made endpoints offered when adding a backend. `custom` intentionally * carries no URL so the user starts from an empty form. @@ -85,6 +99,8 @@ export const BACKEND_PRESETS: readonly BackendPreset[] = [ }, { baseUrl: 'https://api.openai.com', + // newer OpenAI models reject max_tokens + compat: { maxTokensField: 'max_completion_tokens' }, id: 'openai', name: 'OpenAI', protocol: 'openai' diff --git a/tools/ui/src/lib/constants/headers.constants.ts b/tools/ui/src/lib/constants/headers.constants.ts index b717477d02..3f6e1207b7 100644 --- a/tools/ui/src/lib/constants/headers.constants.ts +++ b/tools/ui/src/lib/constants/headers.constants.ts @@ -5,6 +5,8 @@ const MCP_SESSION_ID_VISIBLE_CHARS = 5; export const HEADERS = { /** Anthropic-compatible backends authenticate with this header instead of Authorization */ ANTHROPIC_API_KEY: 'x-api-key', + /** Browser opt-in required by the Anthropic Messages API direct browser access */ + ANTHROPIC_BROWSER_ACCESS: 'anthropic-dangerous-direct-browser-access', /** Required version header for the Anthropic Messages API */ ANTHROPIC_VERSION: 'anthropic-version', /** Canonical casing for the Authorization header (RFC 7235) */ diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index a96b3568fe..8ff2558763 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -33,6 +33,8 @@ import { ReasoningFormat, StreamConnectionState } from '$lib/enums'; +import { getProtocolAdapter } from '$lib/services/protocols'; +import { extractModelName } from '$lib/services/protocols/openai'; import { modelsStore } from '$lib/stores/models/index.svelte'; import { serverStore } from '$lib/stores/server.svelte'; import { settingsStore } from '$lib/stores/settings/index.svelte'; @@ -51,41 +53,6 @@ import { formatAttachmentText } from '$lib/utils/formatters'; import { streamIdentity } from '$lib/utils/stream-identity'; import { buildTimingsFromUsage } from '$lib/utils/timings'; -/** - * llama.cpp-only chat request fields. Strict OpenAI-compatible endpoints - * reject unknown parameters, so they are dropped for those backends. - */ -const COMPAT_ONLY_OMIT_REQUEST_FIELDS = [ - 'add_generation_prompt', - 'backend_sampling', - 'cache_prompt', - 'chat_template_kwargs', - 'continue_final_message', - 'dry_allowed_length', - 'dry_base', - 'dry_multiplier', - 'dry_penalty_last_n', - 'dynatemp_exponent', - 'dynatemp_range', - 'id_slot', - 'min_p', - 'n_keep', - 'n_predict', - 'reasoning_control', - 'reasoning_format', - 'repeat_last_n', - 'repeat_penalty', - 'return_progress', - 'samplers', - 'sse_ping_interval', - 'thinking_budget_tokens', - 'timings_per_token', - 'top_k', - 'typ_p', - 'xtc_probability', - 'xtc_threshold' -]; - interface ResumableStreamState { bytesReceived: number; updatedAt: number; @@ -539,6 +506,8 @@ export class ChatService { let liveTimingsAt = 0; let usage: ApiChatCompletionUsage | undefined; + // the protocol decides how payloads map onto canonical events + const streamReader = getProtocolAdapter(getBackend()).createStreamReader(); const finalizeOpenToolCallBatch = () => { if (!hasOpenToolCallBatch) { return; @@ -578,6 +547,32 @@ export class ChatService { onToolCallChunk?.(serializedToolCalls); } }; + // backends that do not stream their own timings report progress from wall + // clock time and the streamed delta count, throttled to keep updates cheap + const markToken = () => { + firstTokenAt ??= Date.now(); + lastTokenAt = Date.now(); + streamedTokens++; + + if ( + serverStore.capabilities.props || + Date.now() - liveTimingsAt < STREAM_LIVE_TIMINGS_INTERVAL_MS + ) { + return; + } + + liveTimingsAt = Date.now(); + + const liveTimings = buildTimingsFromUsage( + usage, + { firstTokenAt, lastTokenAt, startedAt }, + streamedTokens + ); + + if (liveTimings) { + ChatService.notifyTimings(liveTimings, undefined, onTimings); + } + }; const onVisibilityChange = () => { if (typeof document === 'undefined') return; @@ -679,82 +674,89 @@ export class ChatService { continue; } + let parsed: unknown; + try { - const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); - const choice = parsed.choices?.[0]; - const content = choice?.delta?.content; - const reasoningContent = choice?.delta?.reasoning_content; - const toolCalls = choice?.delta?.tool_calls; - const timings = parsed.timings; - const promptProgress = parsed.prompt_progress; - const chunkUsage = parsed.usage; - const chunkModel = ChatService.extractModelName(parsed); + parsed = JSON.parse(data); + } catch (parseError) { + console.error('Error parsing JSON chunk:', parseError); - if (chunkUsage) usage = chunkUsage; + continue; + } - if (chunkModel && !modelEmitted) { - modelEmitted = true; - onModel?.(chunkModel); - } + for (const event of streamReader.readChunk(parsed)) { + switch (event.type) { + case 'done': + streamFinished = true; - if (parsed.id && !idEmitted) { - idEmitted = true; - onCompletionId?.(parsed.id); - } + break; - if (promptProgress) { - ChatService.notifyTimings(undefined, promptProgress, onTimings); - } + case 'error': + throw new Error(event.message); - if (timings) { - ChatService.notifyTimings(timings, promptProgress, onTimings); - lastTimings = timings; - } - - if (content) { - finalizeOpenToolCallBatch(); - aggregatedContent += content; - - if (!abortSignal?.aborted) { - onChunk?.(content); - } - } - - if (reasoningContent) { - finalizeOpenToolCallBatch(); - fullReasoningContent += reasoningContent; - - if (!abortSignal?.aborted) { - onReasoningChunk?.(reasoningContent); - } - } - - processToolCallDelta(toolCalls); - - if (content || reasoningContent) { - firstTokenAt ??= Date.now(); - lastTokenAt = Date.now(); - streamedTokens++; - - if ( - !serverStore.capabilities.props && - Date.now() - liveTimingsAt >= STREAM_LIVE_TIMINGS_INTERVAL_MS - ) { - liveTimingsAt = Date.now(); - - const liveTimings = buildTimingsFromUsage( - usage, - { firstTokenAt, lastTokenAt, startedAt }, - streamedTokens - ); - - if (liveTimings) { - ChatService.notifyTimings(liveTimings, undefined, onTimings); + case 'id': + if (!idEmitted) { + idEmitted = true; + onCompletionId?.(event.id); } - } + + break; + + case 'model': + if (!modelEmitted) { + modelEmitted = true; + onModel?.(event.model); + } + + break; + + case 'prompt_progress': + ChatService.notifyTimings(undefined, event.progress, onTimings); + + break; + + case 'text': + finalizeOpenToolCallBatch(); + aggregatedContent += event.text; + + if (!abortSignal?.aborted) { + onChunk?.(event.text); + } + + markToken(); + + break; + + case 'thinking': + finalizeOpenToolCallBatch(); + fullReasoningContent += event.text; + + if (!abortSignal?.aborted) { + onReasoningChunk?.(event.text); + } + + markToken(); + + break; + + case 'timings': + ChatService.notifyTimings(event.timings, event.promptProgress, onTimings); + lastTimings = event.timings; + + break; + + case 'tool_calls': + processToolCallDelta(event.deltas); + + break; + + case 'usage': + // providers may split usage across chunks (Anthropic reports input + // tokens on message_start and output tokens on message_delta) + usage = { ...usage, ...event.usage }; + + break; } - } catch (e) { - console.error('Error parsing JSON chunk:', e); } } } @@ -1313,12 +1315,6 @@ export class ChatService { if (timings_per_token !== undefined) requestBody.timings_per_token = timings_per_token; - // OpenAI-compatible servers report token counts in a final usage chunk, which - // the client side timing fallback in handleStreamResponse relies on - if (stream && !serverStore.capabilities.props && requestBody.stream_options === undefined) { - requestBody.stream_options = { include_usage: true }; - } - if (custom) { try { const customParams = typeof custom === 'string' ? JSON.parse(custom) : custom; @@ -1342,10 +1338,17 @@ export class ChatService { ChatService.saveStreamState(conversationId, 0, options.model ?? null); } - ChatService.stripBackendSpecificFields(requestBody); - + // the protocol adapter owns the wire format: it strips llama.cpp-only + // fields, applies the backend's token cap field and adds the usage chunk + const backend = getBackend(); + const wireBody = backend + ? getProtocolAdapter(backend).buildChatRequest( + requestBody as unknown as Record, + backend + ) + : (requestBody as unknown as Record); const response = await fetch(apiChatUrl(), { - body: JSON.stringify(requestBody), + body: JSON.stringify(wireBody), headers, method: 'POST', signal @@ -1490,61 +1493,6 @@ export class ChatService { return apiUrl(`${API_STREAM.BASE}?${query}${offset}`); } - /** - * Extracts model name from Chat Completions API response data. - * Handles various response formats including streaming chunks and final responses. - * - * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name - * in the response. We override it with the actual model name from serverStore. - * - * @param data - Raw response data from the Chat Completions API - * @returns Model name string if found, undefined otherwise - * @private - */ - private static extractModelName(data: unknown): string | undefined { - const asRecord = (value: unknown): Record | undefined => { - return typeof value === 'object' && value !== null - ? (value as Record) - : undefined; - }; - const getTrimmedString = (value: unknown): string | undefined => { - return typeof value === 'string' && value.trim() ? value.trim() : undefined; - }; - const root = asRecord(data); - - if (!root) return undefined; - - // 1) root (some implementations provide `model` at the top level) - const rootModel = getTrimmedString(root.model); - - if (rootModel) { - return rootModel; - } - - // 2) streaming choice (delta) or final response (message) - const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; - - if (!firstChoice) { - return undefined; - } - - // priority: delta.model (first chunk) else message.model (final response) - const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); - - if (deltaModel) { - return deltaModel; - } - - const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); - - if (messageModel) { - return messageModel; - } - - // avoid guessing from non-standard locations (metadata, etc.) - return undefined; - } - /** * Handles non-streaming response from the chat completion API. * Parses the JSON response and extracts the generated content. @@ -1577,7 +1525,7 @@ export class ChatService { } const data: ApiChatCompletionResponse = JSON.parse(responseText); - const responseModel = ChatService.extractModelName(data); + const responseModel = extractModelName(data); if (responseModel) { onModel?.(responseModel); @@ -1739,28 +1687,6 @@ export class ChatService { * Strips legacy inline reasoning content tags from message content. * Handles both plain string content and multipart content arrays. */ - private static stripBackendSpecificFields(body: ApiChatCompletionRequest): void { - const backend = getBackend(); - - if (!backend || backend.protocol === 'llama.cpp') return; - - const record = body as unknown as Record; - - for (const field of COMPAT_ONLY_OMIT_REQUEST_FIELDS) { - delete record[field]; - } - - // compatible endpoints reject the reasoning_content message extension - for (const message of body.messages) { - delete (message as unknown as Record).reasoning_content; - } - - // -1 is llama.cpp's "no limit" sentinel; compatible endpoints reject it - if (typeof body.max_tokens === 'number' && body.max_tokens <= 0) { - delete record.max_tokens; - } - } - private static stripReasoningContent( content: string | ApiChatMessageContentPart[] ): string | ApiChatMessageContentPart[] { diff --git a/tools/ui/src/lib/services/protocols/anthropic.ts b/tools/ui/src/lib/services/protocols/anthropic.ts new file mode 100644 index 0000000000..34281b3110 --- /dev/null +++ b/tools/ui/src/lib/services/protocols/anthropic.ts @@ -0,0 +1,347 @@ +/** + * Anthropic Messages protocol. + * + * The Messages API differs from the OpenAI shape in three ways that matter + * here: the system prompt is a top level field, messages carry typed content + * blocks, and tool calls are content blocks rather than a message field. + */ + +import type { ChatProtocolAdapter, ChatStreamEvent, ChatStreamReader } from './types'; +import { ANTHROPIC_API_VERSION, HEADERS } from '$lib/constants'; +import { ContentPartType, MessageRole } from '$lib/enums'; +import type { Backend } from '$lib/types'; +import type { + ApiChatCompletionTool, + ApiChatCompletionUsage, + ApiChatMessageContentPart, + ApiChatMessageData +} from '$lib/types/api'; + +/** Output cap sent when the user did not configure one; the field is required. */ +const DEFAULT_MAX_TOKENS = 4096; + +interface AnthropicBlock { + type: string; + [key: string]: unknown; +} + +interface AnthropicTool { + description?: string; + input_schema: Record; + name: string; +} + +interface AnthropicStreamEvent { + type?: string; + index?: number; + content_block?: { type?: string; id?: string; name?: string }; + delta?: { type?: string; text?: string; thinking?: string; partial_json?: string }; + error?: { message?: string }; + message?: { + id?: string; + model?: string; + usage?: Record; + }; + usage?: Record; +} + +function authHeaders(backend: Backend): Record { + const headers: Record = { + ...(backend.headers ?? {}), + [HEADERS.ANTHROPIC_BROWSER_ACCESS]: 'true', + [HEADERS.ANTHROPIC_VERSION]: ANTHROPIC_API_VERSION + }; + const apiKey = backend.apiKey?.trim(); + + if (apiKey) { + headers[HEADERS.ANTHROPIC_API_KEY] = apiKey; + } + + return headers; +} + +function textOf(content: string | ApiChatMessageContentPart[]): string { + if (typeof content === 'string') return content; + + return content + .filter((part) => part.type === ContentPartType.TEXT) + .map((part) => part.text ?? '') + .join(''); +} + +function imageBlock(url: string): AnthropicBlock { + const dataUrl = url.match(/^data:([^;,]+);base64,(.*)$/s); + + if (dataUrl) { + return { source: { data: dataUrl[2], media_type: dataUrl[1], type: 'base64' }, type: 'image' }; + } + + return { source: { type: 'url', url }, type: 'image' }; +} + +function userBlocks(content: string | ApiChatMessageContentPart[]): AnthropicBlock[] { + if (typeof content === 'string') { + return content ? [{ text: content, type: 'text' }] : []; + } + + return content.flatMap((part): AnthropicBlock[] => { + if (part.type === ContentPartType.TEXT) { + return part.text ? [{ text: part.text, type: 'text' }] : []; + } + + if (part.type === ContentPartType.IMAGE_URL && part.image_url?.url) { + return [imageBlock(part.image_url.url)]; + } + + // audio and video have no Messages API equivalent + return []; + }); +} + +function assistantBlocks(message: ApiChatMessageData): AnthropicBlock[] { + const blocks: AnthropicBlock[] = []; + const text = textOf(message.content); + + if (text) blocks.push({ text, type: 'text' }); + + for (const call of message.tool_calls ?? []) { + let input: unknown = {}; + + try { + input = call.function?.arguments ? JSON.parse(call.function.arguments) : {}; + } catch { + input = {}; + } + + blocks.push({ id: call.id, input, name: call.function?.name, type: 'tool_use' }); + } + + return blocks; +} + +function convertTools(tools: unknown): AnthropicTool[] | undefined { + if (!Array.isArray(tools) || tools.length === 0) return undefined; + + return (tools as ApiChatCompletionTool[]).map((tool) => ({ + description: tool.function?.description, + input_schema: tool.function?.parameters ?? { properties: {}, type: 'object' }, + name: tool.function?.name + })); +} + +function buildChatRequest( + body: Record, + _backend: Backend +): Record { + const messages = Array.isArray(body.messages) ? (body.messages as ApiChatMessageData[]) : []; + const configuredMaxTokens = typeof body.max_tokens === 'number' ? body.max_tokens : 0; + const maxTokens = configuredMaxTokens > 0 ? configuredMaxTokens : DEFAULT_MAX_TOKENS; + const system: string[] = []; + const converted: { content: AnthropicBlock[]; role: 'assistant' | 'user' }[] = []; + // the Messages API requires strictly alternating roles, so adjacent blocks + // of the same role (tool results, split user turns) are merged + const push = (role: 'assistant' | 'user', content: AnthropicBlock[]): void => { + if (content.length === 0) return; + + const last = converted.at(-1); + + if (last?.role === role) { + last.content.push(...content); + } else { + converted.push({ content, role }); + } + }; + + for (const message of messages) { + if (message.role === MessageRole.SYSTEM) { + const text = textOf(message.content); + + if (text) system.push(text); + + continue; + } + + if (message.role === MessageRole.TOOL) { + push('user', [ + { + content: textOf(message.content), + tool_use_id: message.tool_call_id, + type: 'tool_result' + } + ]); + + continue; + } + + if (message.role === MessageRole.ASSISTANT) { + push('assistant', assistantBlocks(message)); + + continue; + } + + push('user', userBlocks(message.content)); + } + + const request: Record = { + max_tokens: maxTokens, + messages: converted, + model: body.model + }; + + if (system.length > 0) request.system = system.join('\n\n'); + + if (body.stream) request.stream = true; + + if (typeof body.temperature === 'number') request.temperature = body.temperature; + + if (typeof body.top_p === 'number') request.top_p = body.top_p; + + const tools = convertTools(body.tools); + + if (tools) request.tools = tools; + + const chatTemplateKwargs = body.chat_template_kwargs as Record | undefined; + const budgetTokens = + typeof body.thinking_budget_tokens === 'number' ? body.thinking_budget_tokens : 0; + + // extended thinking needs a budget below max_tokens, and Anthropic rejects + // temperature and top_p while thinking is enabled + if ( + chatTemplateKwargs?.enable_thinking === true && + budgetTokens > 0 && + budgetTokens < maxTokens + ) { + request.thinking = { budget_tokens: budgetTokens, type: 'enabled' }; + delete request.temperature; + delete request.top_p; + } + + return request; +} + +/** Map an Anthropic usage object onto the compatible fields, defined keys only. */ +function usageOf(raw: Record | undefined): ApiChatCompletionUsage | undefined { + if (!raw) return undefined; + + const usage: ApiChatCompletionUsage = {}; + + if (typeof raw.input_tokens === 'number') usage.input_tokens = raw.input_tokens; + + if (typeof raw.output_tokens === 'number') usage.output_tokens = raw.output_tokens; + + return Object.keys(usage).length > 0 ? usage : undefined; +} + +function createStreamReader(): ChatStreamReader { + // Anthropic indexes content blocks across text and tool_use, while the + // canonical tool call deltas are indexed among tool calls only + const toolIndexes = new Map(); + + let toolCount = 0; + + const toolIndex = (blockIndex: number | undefined): number => { + if (typeof blockIndex !== 'number') return toolCount++; + + const known = toolIndexes.get(blockIndex); + + if (known !== undefined) return known; + + const index = toolCount++; + + toolIndexes.set(blockIndex, index); + + return index; + }; + + return { + readChunk(payload: unknown): ChatStreamEvent[] { + if (!payload || typeof payload !== 'object') return []; + + const event = payload as AnthropicStreamEvent; + const events: ChatStreamEvent[] = []; + + switch (event.type) { + case 'message_start': { + if (event.message?.id) events.push({ id: event.message.id, type: 'id' }); + + if (event.message?.model) events.push({ model: event.message.model, type: 'model' }); + + const usage = usageOf(event.message?.usage); + + if (usage) events.push({ type: 'usage', usage }); + + break; + } + + case 'content_block_start': { + if (event.content_block?.type === 'tool_use') { + events.push({ + deltas: [ + { + function: { name: event.content_block.name }, + id: event.content_block.id, + index: toolIndex(event.index), + type: 'function' + } + ], + type: 'tool_calls' + }); + } + + break; + } + + case 'content_block_delta': { + const delta = event.delta; + + if (delta?.type === 'text_delta' && delta.text) { + events.push({ text: delta.text, type: 'text' }); + } else if (delta?.type === 'thinking_delta' && delta.thinking) { + events.push({ text: delta.thinking, type: 'thinking' }); + } else if (delta?.type === 'input_json_delta' && delta.partial_json) { + events.push({ + deltas: [ + { + function: { arguments: delta.partial_json }, + index: toolIndex(event.index) + } + ], + type: 'tool_calls' + }); + } + + break; + } + + case 'message_delta': { + const usage = usageOf(event.usage); + + if (usage) events.push({ type: 'usage', usage }); + + break; + } + + case 'message_stop': + events.push({ type: 'done' }); + + break; + + case 'error': + events.push({ + message: event.error?.message ?? 'Anthropic stream error', + type: 'error' + }); + + break; + } + + return events; + } + }; +} + +export const anthropicAdapter: ChatProtocolAdapter = { + authHeaders, + buildChatRequest, + createStreamReader +}; diff --git a/tools/ui/src/lib/services/protocols/index.ts b/tools/ui/src/lib/services/protocols/index.ts new file mode 100644 index 0000000000..97e620d93d --- /dev/null +++ b/tools/ui/src/lib/services/protocols/index.ts @@ -0,0 +1,26 @@ +/** + * Protocol adapter registry. + * + * Resolves the wire mapping for a backend by its protocol. Unknown or + * not-yet-loaded backends fall back to the OpenAI-compatible adapter, which is + * what the local llama-server speaks. + */ + +import { anthropicAdapter } from './anthropic'; +import { openaiAdapter } from './openai'; +import type { ChatProtocolAdapter } from './types'; +import type { Backend, BackendProtocol } from '$lib/types'; + +const ADAPTERS: Record = { + anthropic: anthropicAdapter, + 'llama.cpp': openaiAdapter, + openai: openaiAdapter +}; + +export function getProtocolAdapter(backend?: Backend): ChatProtocolAdapter { + if (!backend) return openaiAdapter; + + return ADAPTERS[backend.protocol] ?? openaiAdapter; +} + +export type { ChatProtocolAdapter, ChatStreamEvent, ChatStreamReader } from './types'; diff --git a/tools/ui/src/lib/services/protocols/openai.ts b/tools/ui/src/lib/services/protocols/openai.ts new file mode 100644 index 0000000000..99fd12af40 --- /dev/null +++ b/tools/ui/src/lib/services/protocols/openai.ts @@ -0,0 +1,183 @@ +/** + * OpenAI-compatible protocol: llama-server, hosted OpenAI endpoints and any + * compatible gateway. llama-server accepts a superset of the wire format, so + * its only specialization is that nothing gets stripped. + */ + +import type { ChatProtocolAdapter, ChatStreamEvent, ChatStreamReader } from './types'; +import { HEADERS } from '$lib/constants'; +import type { Backend } from '$lib/types'; +import type { ApiChatCompletionStreamChunk } from '$lib/types/api'; +import { getBackendCompat } from '$lib/utils/backend'; + +/** + * llama.cpp-only chat request fields. Strict OpenAI-compatible endpoints + * reject unknown parameters, so they are dropped for those backends. + */ +const COMPAT_ONLY_OMIT_REQUEST_FIELDS = [ + 'add_generation_prompt', + 'backend_sampling', + 'cache_prompt', + 'chat_template_kwargs', + 'continue_final_message', + 'dry_allowed_length', + 'dry_base', + 'dry_multiplier', + 'dry_penalty_last_n', + 'dynatemp_exponent', + 'dynatemp_range', + 'id_slot', + 'min_p', + 'n_keep', + 'n_predict', + 'reasoning_control', + 'reasoning_format', + 'repeat_last_n', + 'repeat_penalty', + 'return_progress', + 'samplers', + 'sse_ping_interval', + 'thinking_budget_tokens', + 'timings_per_token', + 'top_k', + 'typ_p', + 'xtc_probability', + 'xtc_threshold' +]; + +function authHeaders(backend: Backend): Record { + const headers: Record = { ...(backend.headers ?? {}) }; + const apiKey = backend.apiKey?.trim(); + + if (apiKey) { + headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`; + } + + return headers; +} + +function buildChatRequest( + body: Record, + backend: Backend +): Record { + if (backend.protocol === 'llama.cpp') return body; + + const compat = getBackendCompat(backend); + const request: Record = { ...body }; + + for (const field of COMPAT_ONLY_OMIT_REQUEST_FIELDS) { + delete request[field]; + } + + // compatible endpoints reject the reasoning_content message extension + const messages = request.messages as { reasoning_content?: string }[] | undefined; + + for (const message of messages ?? []) { + delete message.reasoning_content; + } + + // -1 is llama.cpp's "no limit" sentinel; compatible endpoints reject it + if (typeof request.max_tokens === 'number' && request.max_tokens <= 0) { + delete request.max_tokens; + } + + // newer OpenAI models require max_completion_tokens, most compatible + // endpoints only understand max_tokens + if (compat.maxTokensField === 'max_completion_tokens' && request.max_tokens !== undefined) { + request.max_completion_tokens = request.max_tokens; + delete request.max_tokens; + } + + // a final usage chunk is what the client side timing fallback reads + if (request.stream && compat.supportsUsageInStreaming && request.stream_options === undefined) { + request.stream_options = { include_usage: true }; + } + + return request; +} + +/** + * Model name a payload reports. Streaming chunks carry it on the delta, final + * responses on the message, and some gateways on metadata or the choice itself. + */ +export function extractModelName(data: unknown): string | undefined { + const asRecord = (value: unknown): Record | undefined => + typeof value === 'object' && value !== null ? (value as Record) : undefined; + const getTrimmedString = (value: unknown): string | undefined => + typeof value === 'string' && value.trim() ? value.trim() : undefined; + const root = asRecord(data); + + if (!root) return undefined; + + const rootModel = getTrimmedString(root.model); + + if (rootModel) return rootModel; + + const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; + + if (!firstChoice) return undefined; + + const metadataModel = getTrimmedString(asRecord(firstChoice.metadata)?.model); + + if (metadataModel) return metadataModel; + + const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); + + if (deltaModel) return deltaModel; + + const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); + + if (messageModel) return messageModel; + + return getTrimmedString(firstChoice.model); +} + +function readChunk(payload: unknown): ChatStreamEvent[] { + if (!payload || typeof payload !== 'object') return []; + + const chunk = payload as ApiChatCompletionStreamChunk; + const events: ChatStreamEvent[] = []; + const model = extractModelName(chunk); + + if (chunk.id) events.push({ id: chunk.id, type: 'id' }); + + if (model) events.push({ model, type: 'model' }); + + if (chunk.prompt_progress) { + events.push({ progress: chunk.prompt_progress, type: 'prompt_progress' }); + } + + if (chunk.timings) { + events.push({ + promptProgress: chunk.prompt_progress, + timings: chunk.timings, + type: 'timings' + }); + } + + if (chunk.usage) events.push({ type: 'usage', usage: chunk.usage }); + + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + + if (!delta) return events; + + if (delta.content) events.push({ text: delta.content, type: 'text' }); + + // endpoints disagree on the reasoning field name; take the first one set + const reasoning = delta.reasoning_content ?? delta.reasoning ?? delta.reasoning_text; + + if (reasoning) events.push({ text: reasoning, type: 'thinking' }); + + if (delta.tool_calls) events.push({ deltas: delta.tool_calls, type: 'tool_calls' }); + + return events; +} + +export const openaiAdapter: ChatProtocolAdapter = { + authHeaders, + buildChatRequest, + createStreamReader(): ChatStreamReader { + return { readChunk }; + } +}; diff --git a/tools/ui/src/lib/services/protocols/types.ts b/tools/ui/src/lib/services/protocols/types.ts new file mode 100644 index 0000000000..416a9e3f02 --- /dev/null +++ b/tools/ui/src/lib/services/protocols/types.ts @@ -0,0 +1,42 @@ +/** + * Backend protocol adapters. + * + * A backend speaks one wire protocol. ChatService owns the transport (fetch, + * SSE framing, resume offsets) and delegates the parts that differ per + * protocol here: credential headers, request shaping and stream decoding. + * + * Decoding is per-stream: the Anthropic reader tracks content block state, so + * it must not be shared between concurrent requests. + */ + +import type { Backend } from '$lib/types'; +import type { ApiChatCompletionToolCallDelta, ApiChatCompletionUsage } from '$lib/types/api'; +import type { ChatMessagePromptProgress, ChatMessageTimings } from '$lib/types/chat'; + +/** One canonical delta decoded from a backend's stream payload. */ +export type ChatStreamEvent = + | { type: 'done' } + | { type: 'error'; message: string } + | { type: 'id'; id: string } + | { type: 'model'; model: string } + | { type: 'prompt_progress'; progress: ChatMessagePromptProgress } + | { type: 'text'; text: string } + | { type: 'thinking'; text: string } + | { type: 'timings'; timings: ChatMessageTimings; promptProgress?: ChatMessagePromptProgress } + | { type: 'tool_calls'; deltas: ApiChatCompletionToolCallDelta[] } + | { type: 'usage'; usage: ApiChatCompletionUsage }; + +/** Decodes one stream's payloads. Create one per request. */ +export interface ChatStreamReader { + /** Canonical deltas for one parsed SSE payload. */ + readChunk(payload: unknown): ChatStreamEvent[]; +} + +/** Wire mapping for one protocol. */ +export interface ChatProtocolAdapter { + /** Credential and protocol-required headers for a backend. */ + authHeaders(backend: Backend): Record; + /** Rewrite the canonical request body into this protocol's wire body. */ + buildChatRequest(body: Record, backend: Backend): Record; + createStreamReader(): ChatStreamReader; +} diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts index d1fc52ad20..ebb41d79a6 100644 --- a/tools/ui/src/lib/types/api.d.ts +++ b/tools/ui/src/lib/types/api.d.ts @@ -358,7 +358,9 @@ export interface ApiChatCompletionStreamChunk { metadata?: { model?: string }; delta: { content?: string; + reasoning?: string; reasoning_content?: string; + reasoning_text?: string; model?: string; tool_calls?: ApiChatCompletionToolCallDelta[]; }; diff --git a/tools/ui/src/lib/types/backend.d.ts b/tools/ui/src/lib/types/backend.d.ts index 43b071b4a4..9fe150584d 100644 --- a/tools/ui/src/lib/types/backend.d.ts +++ b/tools/ui/src/lib/types/backend.d.ts @@ -10,6 +10,17 @@ /** Request/response shape a backend speaks. */ export type BackendProtocol = 'llama.cpp' | 'openai' | 'anthropic'; +/** + * Wire-level quirks of a backend's protocol. Capabilities gate llama.cpp + * features; compat describes how the request and stream payloads differ. + */ +export interface BackendCompat { + /** Field carrying the output token cap. */ + maxTokensField: 'max_completion_tokens' | 'max_tokens'; + /** Whether the endpoint accepts stream_options.include_usage. */ + supportsUsageInStreaming: boolean; +} + /** * Features a backend supports. A llama.cpp server exposes extra endpoints on * top of the OpenAI-compatible API; plain OpenAI- and Anthropic-compatible @@ -45,6 +56,8 @@ export interface Backend { baseUrl: string; /** Chat completions path override, e.g. /v1/messages. */ chatPath?: string; + /** Wire quirks overriding the protocol defaults. */ + compat?: Partial; /** Disabled backends stay configured but are not queried. */ enabled: boolean; /** Extra headers merged into every request to this backend. */ @@ -63,6 +76,8 @@ export interface BackendPreset { apiKeyHelp?: string; baseUrl: string; chatPath?: string; + /** Wire quirks this preset needs on top of the protocol defaults. */ + compat?: Partial; id: string; modelsPath?: string; name: string; diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.ts index 56758d957c..3723e8d5f9 100644 --- a/tools/ui/src/lib/types/index.ts +++ b/tools/ui/src/lib/types/index.ts @@ -37,7 +37,13 @@ export type { } from './api'; // Backend types -export type { Backend, BackendCapabilities, BackendPreset, BackendProtocol } from './backend'; +export type { + Backend, + BackendCapabilities, + BackendCompat, + BackendPreset, + BackendProtocol +} from './backend'; // HuggingFace types export type { diff --git a/tools/ui/src/lib/utils/api-headers.ts b/tools/ui/src/lib/utils/api-headers.ts index e71710c133..4938e01aad 100644 --- a/tools/ui/src/lib/utils/api-headers.ts +++ b/tools/ui/src/lib/utils/api-headers.ts @@ -1,7 +1,8 @@ import { getBackend } from './api-base'; import { redactValue } from './redact'; -import { ANTHROPIC_API_VERSION, CORS_PROXY, HEADERS } from '$lib/constants'; +import { CORS_PROXY, HEADERS } from '$lib/constants'; import { MimeTypeApplication } from '$lib/enums'; +import { getProtocolAdapter } from '$lib/services/protocols'; import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { Backend } from '$lib/types'; @@ -23,22 +24,10 @@ export function getAuthHeaders(backendId?: string): Record { /** * Get authorization headers for a backend object, including one that is not * registered yet (used by the connection test on the add-backend form). - * Anthropic-compatible backends authenticate with x-api-key instead of Bearer. + * The protocol adapter owns the credential scheme and any required headers. */ export function getAuthHeadersForBackend(backend: Backend): Record { - const headers: Record = { ...(backend.headers ?? {}) }; - const apiKey = backend.apiKey?.trim(); - - if (!apiKey) return headers; - - if (backend.protocol === 'anthropic') { - headers[HEADERS.ANTHROPIC_API_KEY] = apiKey; - headers[HEADERS.ANTHROPIC_VERSION] = ANTHROPIC_API_VERSION; - } else { - headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`; - } - - return headers; + return getProtocolAdapter(backend).authHeaders(backend); } /** diff --git a/tools/ui/src/lib/utils/backend.ts b/tools/ui/src/lib/utils/backend.ts index d00a37b86d..2722d62f9a 100644 --- a/tools/ui/src/lib/utils/backend.ts +++ b/tools/ui/src/lib/utils/backend.ts @@ -8,13 +8,14 @@ import { BACKEND_CAPABILITIES, + BACKEND_COMPAT, BACKEND_ID_PREFIX, BACKEND_PROTOCOLS, DEFAULT_BACKEND_CHAT_PATH, DEFAULT_BACKEND_MODELS_PATH, LOCAL_BACKEND_ID } from '$lib/constants'; -import type { Backend, BackendCapabilities, BackendProtocol } from '$lib/types'; +import type { Backend, BackendCapabilities, BackendCompat, BackendProtocol } from '$lib/types'; /** Absolute chat completions URL for a backend. */ export function backendChatUrl(backend: Backend): string { @@ -31,6 +32,11 @@ export function getBackendCapabilities(backend: Backend): BackendCapabilities { return BACKEND_CAPABILITIES[backend.protocol] ?? BACKEND_CAPABILITIES.openai; } +/** Wire quirks for a backend: protocol defaults overridden by the backend. */ +export function getBackendCompat(backend: Backend): BackendCompat { + return { ...(BACKEND_COMPAT[backend.protocol] ?? BACKEND_COMPAT.openai), ...backend.compat }; +} + /** The built-in backend pointing at the server that serves this UI. */ export function createLocalBackend(apiKey?: string, enabled = true): Backend { return { @@ -107,6 +113,7 @@ function parseBackendEntry(entry: unknown, index: number): Backend | null { apiKey, baseUrl, chatPath: parseOptionalPath(raw.chatPath), + compat: parseBackendCompat(raw.compat, protocol), enabled: raw.enabled !== false, headers: parseBackendHeaders(raw.headers), id, @@ -116,6 +123,36 @@ function parseBackendEntry(entry: unknown, index: number): Backend | null { }; } +// only keep the override keys the protocol understands, so a stale persisted +// value can never inject an unknown field into a request +function parseBackendCompat( + raw: unknown, + protocol: BackendProtocol +): Partial | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + + const entry = raw as Record; + const defaults = BACKEND_COMPAT[protocol] ?? BACKEND_COMPAT.openai; + const overrides: Partial = {}; + + if (entry.maxTokensField === 'max_tokens' || entry.maxTokensField === 'max_completion_tokens') { + overrides.maxTokensField = entry.maxTokensField; + } + + if (typeof entry.supportsUsageInStreaming === 'boolean') { + overrides.supportsUsageInStreaming = entry.supportsUsageInStreaming; + } + + // drop a no-op override so an unmodified backend stays undefined + const isDefault = + (overrides.maxTokensField === undefined || + overrides.maxTokensField === defaults.maxTokensField) && + (overrides.supportsUsageInStreaming === undefined || + overrides.supportsUsageInStreaming === defaults.supportsUsageInStreaming); + + return isDefault ? undefined : overrides; +} + function parseBackendHeaders(raw: unknown): Record | undefined { if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined;