mirror of
https://github.com/mostlygeek/llama-swap.git
synced 2026-06-09 06:46:34 +02:00
proxy: add setParamsByID support for peer models
Extend the peer proxy to support setParamsByID filters, matching the
existing capability for local models. The ${MODEL_ID} macro in
setParamsByID keys is expanded per-model so a single peer config
covering multiple models generates per-model aliases.
- Expand global macros in peer filters.setParamsByID during config load
- Validate setParamsByID values for unknown macros; allow ${MODEL_ID} in keys
- Build per-model expanded filters and alias map in NewPeerProxy
- Add RealPeerModelName to resolve aliases to base model IDs
- Update HasPeerModel to recognise setParamsByID aliases
- Rewrite model field in request body to base model ID when alias used
- Apply setParamsByID params in proxyInferenceHandler for peers
- Sync alias resolution to proxyOAIPostFormHandler and proxyGETModelHandler
fixes #697
This commit is contained in:
@@ -521,6 +521,25 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
|
||||
}
|
||||
peerConfig.Filters.SetParams = result.(map[string]any)
|
||||
}
|
||||
|
||||
// Substitute in setParamsByID keys and values (type-preserving)
|
||||
// Note: ${MODEL_ID} in keys is intentional and will be expanded per-model later
|
||||
if len(peerConfig.Filters.SetParamsByID) > 0 {
|
||||
newSetParamsByID := make(map[string]map[string]any, len(peerConfig.Filters.SetParamsByID))
|
||||
for key, paramMap := range peerConfig.Filters.SetParamsByID {
|
||||
newKey := strings.ReplaceAll(key, macroSlug, macroStr)
|
||||
newValAny, err := substituteMacroInValue(any(paramMap), entry.Name, entry.Value)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("peers.%s.filters.setParamsByID: %w", peerName, err)
|
||||
}
|
||||
newParamMap, ok := newValAny.(map[string]any)
|
||||
if !ok {
|
||||
return Config{}, fmt.Errorf("peers.%s.filters.setParamsByID: unexpected type after macro substitution", peerName)
|
||||
}
|
||||
newSetParamsByID[newKey] = newParamMap
|
||||
}
|
||||
peerConfig.Filters.SetParamsByID = newSetParamsByID
|
||||
}
|
||||
}
|
||||
|
||||
// Validate no unknown macros remain
|
||||
@@ -535,6 +554,19 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
|
||||
return Config{}, err
|
||||
}
|
||||
}
|
||||
// Validate setParamsByID: values must have no unknown macros; keys may contain ${MODEL_ID}
|
||||
for key, paramMap := range peerConfig.Filters.SetParamsByID {
|
||||
if matches := macroPatternRegex.FindAllStringSubmatch(key, -1); len(matches) > 0 {
|
||||
for _, match := range matches {
|
||||
if match[1] != "MODEL_ID" {
|
||||
return Config{}, fmt.Errorf("peers.%s.filters.setParamsByID: unknown macro '${%s}' in key", peerName, match[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := validateNestedForUnknownMacros(any(paramMap), fmt.Sprintf("peers.%s.filters.setParamsByID[%s]", peerName, key)); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
}
|
||||
config.Peers[peerName] = peerConfig
|
||||
}
|
||||
|
||||
|
||||
+78
-11
@@ -20,8 +20,10 @@ type peerProxyMember struct {
|
||||
}
|
||||
|
||||
type PeerProxy struct {
|
||||
peers config.PeerDictionaryConfig
|
||||
proxyMap map[string]*peerProxyMember
|
||||
peers config.PeerDictionaryConfig
|
||||
proxyMap map[string]*peerProxyMember
|
||||
peerAliases map[string]string // alias → base model ID (from setParamsByID keys)
|
||||
modelFilters map[string]config.Filters // base model ID → filters with ${MODEL_ID} expanded
|
||||
}
|
||||
|
||||
func NewPeerProxy(peers config.PeerDictionaryConfig, proxyLogger *LogMonitor) (*PeerProxy, error) {
|
||||
@@ -97,29 +99,94 @@ func NewPeerProxy(peers config.PeerDictionaryConfig, proxyLogger *LogMonitor) (*
|
||||
}
|
||||
}
|
||||
|
||||
// Build per-model expanded filters and alias map from setParamsByID keys
|
||||
peerAliases := make(map[string]string)
|
||||
modelFilters := make(map[string]config.Filters)
|
||||
|
||||
for _, peerID := range peerIDs {
|
||||
peer := peers[peerID]
|
||||
for _, modelID := range peer.Models {
|
||||
if _, found := proxyMap[modelID]; !found {
|
||||
continue // model was skipped as duplicate
|
||||
}
|
||||
expanded := expandPeerFiltersForModel(peer.Filters, modelID)
|
||||
modelFilters[modelID] = expanded
|
||||
|
||||
for key := range expanded.SetParamsByID {
|
||||
if key == modelID {
|
||||
continue
|
||||
}
|
||||
if _, exists := proxyMap[key]; exists {
|
||||
proxyLogger.Warnf("peer %s: setParamsByID key '%s' conflicts with an existing model, skipping alias", peerID, key)
|
||||
continue
|
||||
}
|
||||
if existingModel, exists := peerAliases[key]; exists {
|
||||
if existingModel != modelID {
|
||||
proxyLogger.Warnf("peer %s: duplicate setParamsByID alias '%s' already registered for model %s, skipping", peerID, key, existingModel)
|
||||
}
|
||||
continue
|
||||
}
|
||||
peerAliases[key] = modelID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &PeerProxy{
|
||||
peers: peers,
|
||||
proxyMap: proxyMap,
|
||||
peers: peers,
|
||||
proxyMap: proxyMap,
|
||||
peerAliases: peerAliases,
|
||||
modelFilters: modelFilters,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// expandPeerFiltersForModel returns a copy of f with ${MODEL_ID} replaced by modelID in setParamsByID keys.
|
||||
func expandPeerFiltersForModel(f config.Filters, modelID string) config.Filters {
|
||||
if len(f.SetParamsByID) == 0 {
|
||||
return f
|
||||
}
|
||||
expanded := config.Filters{
|
||||
StripParams: f.StripParams,
|
||||
SetParams: f.SetParams,
|
||||
}
|
||||
expanded.SetParamsByID = make(map[string]map[string]any, len(f.SetParamsByID))
|
||||
const modelIDMacro = "${MODEL_ID}"
|
||||
for key, params := range f.SetParamsByID {
|
||||
newKey := strings.ReplaceAll(key, modelIDMacro, modelID)
|
||||
expanded.SetParamsByID[newKey] = params
|
||||
}
|
||||
return expanded
|
||||
}
|
||||
|
||||
func (p *PeerProxy) HasPeerModel(modelID string) bool {
|
||||
_, found := p.proxyMap[modelID]
|
||||
if _, found := p.proxyMap[modelID]; found {
|
||||
return true
|
||||
}
|
||||
_, found := p.peerAliases[modelID]
|
||||
return found
|
||||
}
|
||||
|
||||
// GetPeerFilters returns the filters for a peer model, or empty filters if not found
|
||||
// RealPeerModelName resolves an alias or model ID to the base model ID.
|
||||
func (p *PeerProxy) RealPeerModelName(modelID string) (string, bool) {
|
||||
if _, found := p.proxyMap[modelID]; found {
|
||||
return modelID, true
|
||||
}
|
||||
if realID, found := p.peerAliases[modelID]; found {
|
||||
return realID, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// GetPeerFilters returns the expanded filters for the given model ID (or alias).
|
||||
func (p *PeerProxy) GetPeerFilters(modelID string) config.Filters {
|
||||
pp, found := p.proxyMap[modelID]
|
||||
realID, found := p.RealPeerModelName(modelID)
|
||||
if !found {
|
||||
return config.Filters{}
|
||||
}
|
||||
// Get the peer config using the peerID
|
||||
peer, found := p.peers[pp.peerID]
|
||||
if !found {
|
||||
filters, ok := p.modelFilters[realID]
|
||||
if !ok {
|
||||
return config.Filters{}
|
||||
}
|
||||
return peer.Filters
|
||||
return filters
|
||||
}
|
||||
|
||||
func (p *PeerProxy) ListPeers() config.PeerDictionaryConfig {
|
||||
|
||||
@@ -309,3 +309,124 @@ func TestNewPeerProxy_CustomTimeouts(t *testing.T) {
|
||||
// ForceAttemptHTTP2 should be enabled
|
||||
assert.True(t, transport.ForceAttemptHTTP2)
|
||||
}
|
||||
|
||||
func TestPeerProxy_SetParamsByID_AliasRegistration(t *testing.T) {
|
||||
proxyURL, _ := url.Parse("http://peer1.example.com:8080")
|
||||
peers := config.PeerDictionaryConfig{
|
||||
"peer1": config.PeerConfig{
|
||||
Proxy: "http://peer1.example.com:8080",
|
||||
ProxyURL: proxyURL,
|
||||
Models: []string{"model-a", "model-b"},
|
||||
Filters: config.Filters{
|
||||
SetParamsByID: map[string]map[string]any{
|
||||
"model-a:nothink": {"enable_thinking": false},
|
||||
"model-a:high": {"reasoning_effort": "high"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pp, err := NewPeerProxy(peers, testLogger)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Base models accessible directly
|
||||
assert.True(t, pp.HasPeerModel("model-a"))
|
||||
assert.True(t, pp.HasPeerModel("model-b"))
|
||||
|
||||
// Aliases accessible via HasPeerModel
|
||||
assert.True(t, pp.HasPeerModel("model-a:nothink"))
|
||||
assert.True(t, pp.HasPeerModel("model-a:high"))
|
||||
|
||||
// Non-existent alias
|
||||
assert.False(t, pp.HasPeerModel("model-a:unknown"))
|
||||
}
|
||||
|
||||
func TestPeerProxy_SetParamsByID_ModelIDMacroExpansion(t *testing.T) {
|
||||
proxyURL, _ := url.Parse("http://peer1.example.com:8080")
|
||||
peers := config.PeerDictionaryConfig{
|
||||
"peer1": config.PeerConfig{
|
||||
Proxy: "http://peer1.example.com:8080",
|
||||
ProxyURL: proxyURL,
|
||||
Models: []string{"model-a", "model-b"},
|
||||
Filters: config.Filters{
|
||||
SetParamsByID: map[string]map[string]any{
|
||||
"${MODEL_ID}:nothink": {"enable_thinking": false},
|
||||
"${MODEL_ID}:high": {"reasoning_effort": "high"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pp, err := NewPeerProxy(peers, testLogger)
|
||||
require.NoError(t, err)
|
||||
|
||||
// ${MODEL_ID} expanded to "model-a"
|
||||
assert.True(t, pp.HasPeerModel("model-a:nothink"))
|
||||
assert.True(t, pp.HasPeerModel("model-a:high"))
|
||||
|
||||
// ${MODEL_ID} expanded to "model-b"
|
||||
assert.True(t, pp.HasPeerModel("model-b:nothink"))
|
||||
assert.True(t, pp.HasPeerModel("model-b:high"))
|
||||
}
|
||||
|
||||
func TestPeerProxy_RealPeerModelName(t *testing.T) {
|
||||
proxyURL, _ := url.Parse("http://peer1.example.com:8080")
|
||||
peers := config.PeerDictionaryConfig{
|
||||
"peer1": config.PeerConfig{
|
||||
Proxy: "http://peer1.example.com:8080",
|
||||
ProxyURL: proxyURL,
|
||||
Models: []string{"model-a"},
|
||||
Filters: config.Filters{
|
||||
SetParamsByID: map[string]map[string]any{
|
||||
"${MODEL_ID}:nothink": {"enable_thinking": false},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pp, err := NewPeerProxy(peers, testLogger)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Base model resolves to itself
|
||||
realID, found := pp.RealPeerModelName("model-a")
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, "model-a", realID)
|
||||
|
||||
// Alias resolves to base model
|
||||
realID, found = pp.RealPeerModelName("model-a:nothink")
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, "model-a", realID)
|
||||
|
||||
// Unknown model
|
||||
_, found = pp.RealPeerModelName("unknown")
|
||||
assert.False(t, found)
|
||||
}
|
||||
|
||||
func TestPeerProxy_GetPeerFilters_WithAlias(t *testing.T) {
|
||||
proxyURL, _ := url.Parse("http://peer1.example.com:8080")
|
||||
peers := config.PeerDictionaryConfig{
|
||||
"peer1": config.PeerConfig{
|
||||
Proxy: "http://peer1.example.com:8080",
|
||||
ProxyURL: proxyURL,
|
||||
Models: []string{"model-a"},
|
||||
Filters: config.Filters{
|
||||
SetParams: map[string]any{"temperature": 0.7},
|
||||
SetParamsByID: map[string]map[string]any{
|
||||
"${MODEL_ID}:nothink": {"enable_thinking": false},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pp, err := NewPeerProxy(peers, testLogger)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Filters for base model
|
||||
filters := pp.GetPeerFilters("model-a")
|
||||
assert.Equal(t, map[string]any{"temperature": 0.7}, filters.SetParams)
|
||||
assert.Contains(t, filters.SetParamsByID, "model-a:nothink")
|
||||
|
||||
// Filters for alias point to same expanded config
|
||||
filtersAlias := pp.GetPeerFilters("model-a:nothink")
|
||||
assert.Equal(t, filters, filtersAlias)
|
||||
}
|
||||
|
||||
+31
-3
@@ -781,7 +781,19 @@ func (pm *ProxyManager) proxyInferenceHandler(c *gin.Context) {
|
||||
nextHandler = localHandler
|
||||
} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
|
||||
pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
|
||||
modelID = requestedModel
|
||||
|
||||
// Resolve alias to base model ID (e.g. "model_a:nothink" → "model_a")
|
||||
realPeerModelID, _ := pm.peerProxy.RealPeerModelName(requestedModel)
|
||||
modelID = realPeerModelID
|
||||
|
||||
// Rewrite model field to base model ID so the peer server receives the correct name
|
||||
if realPeerModelID != requestedModel {
|
||||
bodyBytes, err = sjson.SetBytes(bodyBytes, "model", realPeerModelID)
|
||||
if err != nil {
|
||||
pm.sendErrorResponse(c, http.StatusInternalServerError, "error rewriting model name in JSON")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// issue #453 apply filters for peer requests
|
||||
peerFilters := pm.peerProxy.GetPeerFilters(requestedModel)
|
||||
@@ -808,6 +820,17 @@ func (pm *ProxyManager) proxyInferenceHandler(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// setParamsByID: set params based on the requested model ID (runs after setParams, can override it)
|
||||
setParamsByIDParams, setParamsByIDKeys := peerFilters.SanitizedSetParamsByID(requestedModel)
|
||||
for _, key := range setParamsByIDKeys {
|
||||
pm.proxyLogger.Debugf("<%s> setting param by id: %s", requestedModel, key)
|
||||
bodyBytes, err = sjson.SetBytes(bodyBytes, key, setParamsByIDParams[key])
|
||||
if err != nil {
|
||||
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error setting parameter %s in request", key))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
nextHandler = pm.peerProxy.ProxyRequest
|
||||
}
|
||||
|
||||
@@ -879,7 +902,11 @@ func (pm *ProxyManager) proxyOAIPostFormHandler(c *gin.Context) {
|
||||
pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
|
||||
} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
|
||||
pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
|
||||
modelID = requestedModel
|
||||
realPeerModelID, _ := pm.peerProxy.RealPeerModelName(requestedModel)
|
||||
modelID = realPeerModelID
|
||||
if realPeerModelID != requestedModel {
|
||||
useModelName = realPeerModelID // rewrite model field in reconstructed form
|
||||
}
|
||||
nextHandler = pm.peerProxy.ProxyRequest
|
||||
}
|
||||
|
||||
@@ -1000,7 +1027,8 @@ func (pm *ProxyManager) proxyGETModelHandler(c *gin.Context) {
|
||||
}
|
||||
pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
|
||||
} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
|
||||
modelID = requestedModel
|
||||
realPeerModelID, _ := pm.peerProxy.RealPeerModelName(requestedModel)
|
||||
modelID = realPeerModelID
|
||||
pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
|
||||
nextHandler = pm.peerProxy.ProxyRequest
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
@@ -1641,6 +1642,118 @@ models:
|
||||
})
|
||||
}
|
||||
|
||||
func TestProxyManager_PeerProxy_SetParamsByID(t *testing.T) {
|
||||
t.Run("setParamsByID alias routes to peer and applies params", func(t *testing.T) {
|
||||
var receivedBody string
|
||||
peerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
receivedBody = string(body)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(`{"response":"ok"}`))
|
||||
}))
|
||||
defer peerServer.Close()
|
||||
|
||||
testConfig := testConfigFromYAML(t, fmt.Sprintf(`
|
||||
logLevel: error
|
||||
peers:
|
||||
test-peer:
|
||||
proxy: %s
|
||||
models:
|
||||
- peer-model
|
||||
filters:
|
||||
setParams:
|
||||
temperature: 0.5
|
||||
setParamsByID:
|
||||
"${MODEL_ID}:nothink":
|
||||
enable_thinking: false
|
||||
"${MODEL_ID}:high":
|
||||
reasoning_effort: high
|
||||
models:
|
||||
local-model:
|
||||
cmd: {{RESPONDER}} --port ${PORT} --silent --respond local-model
|
||||
`, peerServer.URL))
|
||||
|
||||
proxy := New(testConfig)
|
||||
defer proxy.StopProcesses(StopImmediately)
|
||||
injectTestHandlers(proxy, nil)
|
||||
|
||||
tests := []struct {
|
||||
requestedModel string
|
||||
wantModel string
|
||||
wantThinking string
|
||||
wantEffort string
|
||||
wantTemperature string
|
||||
}{
|
||||
// base model: only setParams applied
|
||||
{requestedModel: "peer-model", wantModel: "peer-model", wantTemperature: "0.5", wantThinking: "", wantEffort: ""},
|
||||
// alias: setParams + setParamsByID[nothink] applied, model rewritten
|
||||
{requestedModel: "peer-model:nothink", wantModel: "peer-model", wantTemperature: "0.5", wantThinking: "false", wantEffort: ""},
|
||||
// alias: setParams + setParamsByID[high] applied, model rewritten
|
||||
{requestedModel: "peer-model:high", wantModel: "peer-model", wantTemperature: "0.5", wantThinking: "", wantEffort: "high"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.requestedModel, func(t *testing.T) {
|
||||
receivedBody = ""
|
||||
reqBody := fmt.Sprintf(`{"model":%q}`, tt.requestedModel)
|
||||
req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
|
||||
w := CreateTestResponseRecorder()
|
||||
proxy.ServeHTTP(w, req)
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
gotModel := gjson.Get(receivedBody, "model").String()
|
||||
assert.Equal(t, tt.wantModel, gotModel, "model field mismatch")
|
||||
|
||||
gotTemperature := gjson.Get(receivedBody, "temperature").String()
|
||||
assert.Equal(t, tt.wantTemperature, gotTemperature, "temperature mismatch")
|
||||
|
||||
gotThinking := gjson.Get(receivedBody, "enable_thinking").String()
|
||||
assert.Equal(t, tt.wantThinking, gotThinking, "enable_thinking mismatch")
|
||||
|
||||
gotEffort := gjson.Get(receivedBody, "reasoning_effort").String()
|
||||
assert.Equal(t, tt.wantEffort, gotEffort, "reasoning_effort mismatch")
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("peer alias not accessible as local model alias", func(t *testing.T) {
|
||||
peerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(`{"response":"ok"}`))
|
||||
}))
|
||||
defer peerServer.Close()
|
||||
|
||||
testConfig := testConfigFromYAML(t, fmt.Sprintf(`
|
||||
logLevel: error
|
||||
peers:
|
||||
test-peer:
|
||||
proxy: %s
|
||||
models:
|
||||
- peer-model
|
||||
filters:
|
||||
setParamsByID:
|
||||
"peer-model:nothink":
|
||||
enable_thinking: false
|
||||
models:
|
||||
local-model:
|
||||
cmd: {{RESPONDER}} --port ${PORT} --silent --respond local-model
|
||||
`, peerServer.URL))
|
||||
|
||||
proxy := New(testConfig)
|
||||
defer proxy.StopProcesses(StopImmediately)
|
||||
injectTestHandlers(proxy, nil)
|
||||
|
||||
// The peer alias should work
|
||||
reqBody := `{"model":"peer-model:nothink"}`
|
||||
req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
|
||||
w := CreateTestResponseRecorder()
|
||||
proxy.ServeHTTP(w, req)
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
})
|
||||
}
|
||||
|
||||
func TestProxyManager_SdApiTxt2ImgRouting(t *testing.T) {
|
||||
conf := testConfigFromYAML(t, `
|
||||
healthCheckTimeout: 15
|
||||
|
||||
Reference in New Issue
Block a user