wylab/llama-swap-rocm
1
0
Fork 0
forked from wylab/llama-swap
Code Pull Requests Actions Packages Activity

proxy,proxy/config: restore timeouts to pre PR 619 (#648)

Browse Source 
Reset the default ResponseHeader timeout to 0 (no timeout) which was set
to 60 seconds in PR #619.

Fixes #647
This commit is contained in:
Benson Wong
2026-04-11 20:42:13 -07:00
committed by GitHub
parent 7b2b82777f
commit a9d840ffd7
9 changed files with 74 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config-schema.json
+23 -11
View File
@@ -47,31 +47,37 @@
"type": "integer",
"minimum": 0,
"default": 30,
"description": "TCP connection timeout in seconds. Set to 0 to disable (not recommended)."
"description": "TCP connection timeout in seconds. Set to 0 to disable."
},
"keepalive": {
"type": "integer",
"minimum": 0,
"default": 30,
"description": "TCP keepalive timeout in seconds. Set to 0 to disable."
},
"responseHeader": {
"type": "integer",
"minimum": 0,
"default": 60,
"description": "Time to wait for response headers in seconds. Set to 0 to disable (not recommended)."
"default": 0,
"description": "Time to wait for response headers in seconds. Set to 0 to disable."
},
"tlsHandshake": {
"type": "integer",
"minimum": 0,
"default": 10,
"description": "TLS handshake timeout in seconds. Set to 0 to disable (not recommended)."
"description": "TLS handshake timeout in seconds. Set to 0 to disable."
},
"expectContinue": {
"type": "integer",
"minimum": 0,
"default": 1,
"description": "Expect-Continue timeout in seconds. Set to 0 to disable (not recommended)."
"description": "Expect-Continue timeout in seconds. Set to 0 to disable."
},
"idleConn": {
"type": "integer",
"minimum": 0,
"default": 90,
"description": "Idle connection timeout in seconds. Set to 0 to disable (not recommended)."
"description": "Idle connection timeout in seconds. Set to 0 to disable."
}
},
"additionalProperties": false,
@@ -413,25 +419,31 @@
"properties": {
"connect": {
"type": "integer",
"minimum": 1,
"minimum": 0,
"default": 30,
"description": "TCP connection timeout in seconds."
},
"keepalive": {
"type": "integer",
"minimum": 0,
"default": 30,
"description": "TCP keepalive connection timeout in seconds."
},
"responseHeader": {
"type": "integer",
"minimum": 1,
"default": 60,
"minimum": 0,
"default": 0,
"description": "Time to wait for response headers in seconds."
},
"tlsHandshake": {
"type": "integer",
"minimum": 1,
"minimum": 0,
"default": 10,
"description": "TLS handshake timeout in seconds."
},
"idleConn": {
"type": "integer",
"minimum": 1,
"minimum": 0,
"default": 90,
"description": "Idle connection timeout in seconds."
}
config.example.yaml
+7 -5
View File
@@ -287,14 +287,15 @@ models:
# timeouts: configure proxy connection timeouts for this model
# - optional, defaults shown below
# - useful for models running on slower hardware that need longer timeouts
# - connect: TCP connection timeout in seconds
# - responseHeader: time to wait for response headers in seconds
# (increasing this helps avoid 502 errors on slow hardware)
# - tlsHandshake: TLS handshake timeout in seconds
# - idleConn: idle connection timeout in seconds
# - connect: TCP dial connection timeout in seconds, default: 30 seconds
# - keepalive: TCP connection keepalive timeout, default: 30 seconds
# - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
# - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
# - idleConn: idle connection timeout in seconds, default: 90 seconds
# - set any value to 0 to disable that timeout (not recommended)
timeouts:
connect: 30
keepalive: 0
responseHeader: 60
tlsHandshake: 10
idleConn: 90
@@ -447,6 +448,7 @@ peers:
# - set any value to 0 to disable that timeout (not recommended)
timeouts:
connect: 30
keepalive: 30
responseHeader: 60
tlsHandshake: 10
idleConn: 90
proxy/config/config_posix_test.go
+13 -28
View File
@@ -163,6 +163,15 @@ groups:
modelLoadingState := false
defaultTimeout := TimeoutsConfig{
Connect: 30,
KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
}
expected := Config{
LogLevel: "info",
LogTimeFormat: "",
@@ -187,13 +196,7 @@ groups:
Name: "Model 1",
Description: "This is model 1",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
Timeouts: defaultTimeout,
},
"model2": {
Cmd: "path/to/server --arg1 one",
@@ -202,13 +205,7 @@ groups:
Env: []string{},
CheckEndpoint: "/",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
Timeouts: defaultTimeout,
},
"model3": {
Cmd: "path/to/cmd --arg1 one",
@@ -217,13 +214,7 @@ groups:
Env: []string{},
CheckEndpoint: "/",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
Timeouts: defaultTimeout,
},
"model4": {
Cmd: "path/to/cmd --arg1 one",
@@ -232,13 +223,7 @@ groups:
Aliases: []string{},
Env: []string{},
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
Timeouts: defaultTimeout,
},
},
HealthCheckTimeout: 15,
proxy/config/config_test.go
+1 -1
View File
@@ -1475,7 +1475,7 @@ models:
// Default values should be set during unmarshaling
assert.Equal(t, 30, modelConfig.Timeouts.Connect)
assert.Equal(t, 60, modelConfig.Timeouts.ResponseHeader)
assert.Equal(t, 0, modelConfig.Timeouts.ResponseHeader)
assert.Equal(t, 10, modelConfig.Timeouts.TLSHandshake)
assert.Equal(t, 1, modelConfig.Timeouts.ExpectContinue)
assert.Equal(t, 90, modelConfig.Timeouts.IdleConn)
proxy/config/config_windows_test.go
+13 -28
View File
@@ -155,6 +155,15 @@ groups:
modelLoadingState := false
defaultTimeout := TimeoutsConfig{
Connect: 30,
KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
}
expected := Config{
LogLevel: "info",
LogTimeFormat: "",
@@ -173,13 +182,7 @@ groups:
Env: []string{"VAR1=value1", "VAR2=value2"},
CheckEndpoint: "/health",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
Timeouts: defaultTimeout,
},
"model2": {
Cmd: "path/to/server --arg1 one",
@@ -189,13 +192,7 @@ groups:
Env: []string{},
CheckEndpoint: "/",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
Timeouts: defaultTimeout,
},
"model3": {
Cmd: "path/to/cmd --arg1 one",
@@ -205,13 +202,7 @@ groups:
Env: []string{},
CheckEndpoint: "/",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
Timeouts: defaultTimeout,
},
"model4": {
Cmd: "path/to/cmd --arg1 one",
@@ -221,13 +212,7 @@ groups:
Aliases: []string{},
Env: []string{},
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
Timeouts: defaultTimeout,
},
},
HealthCheckTimeout: 15,
proxy/config/model_config.go
+11 -6
View File
@@ -10,12 +10,14 @@ const (
)
// TimeoutsConfig holds timeout settings for proxy connections
// 0 = no timeout
type TimeoutsConfig struct {
Connect int `yaml:"connect"` // seconds, 0 = no timeout (not recommended)
ResponseHeader int `yaml:"responseHeader"` // seconds, 0 = no timeout (not recommended)
TLSHandshake int `yaml:"tlsHandshake"` // seconds, 0 = no timeout (not recommended)
ExpectContinue int `yaml:"expectContinue"` // seconds, 0 = no timeout (not recommended)
IdleConn int `yaml:"idleConn"` // seconds, 0 = no timeout (not recommended)
Connect int `yaml:"connect"`
KeepAlive int `yaml:"keepalive"`
ResponseHeader int `yaml:"responseHeader"`
TLSHandshake int `yaml:"tlsHandshake"`
ExpectContinue int `yaml:"expectContinue"`
IdleConn int `yaml:"idleConn"`
}
type ModelConfig struct {
@@ -69,9 +71,12 @@ func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
ConcurrencyLimit: 0,
Name: "",
Description: "",
// matches http.DefaultTransport
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
proxy/config/peer.go
+4
View File
@@ -24,8 +24,12 @@ func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
ApiKey: "",
Models: []string{},
Filters: Filters{},
// mostly matches http.DefaultTransport but with a 60s ResponseHeader timeout
// to match the pre PR #619 functionality
Timeouts: TimeoutsConfig{
Connect: 30,
KeepAlive: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
proxy/peerproxy.go
+1 -1
View File
@@ -42,7 +42,7 @@ func NewPeerProxy(peers config.PeerDictionaryConfig, proxyLogger *LogMonitor) (*
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: time.Duration(peer.Timeouts.Connect) * time.Second,
KeepAlive: 30 * time.Second,
KeepAlive: time.Duration(peer.Timeouts.KeepAlive) * time.Second,
}).DialContext,
TLSHandshakeTimeout: time.Duration(peer.Timeouts.TLSHandshake) * time.Second,
ResponseHeaderTimeout: time.Duration(peer.Timeouts.ResponseHeader) * time.Second,
proxy/process.go
+1 -1
View File
@@ -102,7 +102,7 @@ func NewProcess(ID string, healthCheckTimeout int, config config.ModelConfig, pr
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: time.Duration(config.Timeouts.Connect) * time.Second,
KeepAlive: 30 * time.Second,
KeepAlive: time.Duration(config.Timeouts.KeepAlive) * time.Second,
}).DialContext,
TLSHandshakeTimeout: time.Duration(config.Timeouts.TLSHandshake) * time.Second,
ResponseHeaderTimeout: time.Duration(config.Timeouts.ResponseHeader) * time.Second,