From 2cd6202a2d1192cd0901f84a20cf3920bf24b8b4 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Wed, 1 Jul 2026 03:44:48 +0000 Subject: [PATCH 01/11] session boost timeout Signed-off-by: YaoZengzeng --- charts/kthena/charts/networking/README.md | 26 +++++----- .../kthena-router/component/deployment.yaml | 6 +++ charts/kthena/charts/networking/values.yaml | 8 +++ charts/kthena/values.yaml | 8 +++ .../docs/reference/helm-chart-values.md | 2 + docs/kthena/docs/user-guide/session-boost.md | 45 +++++++++++++--- docs/proposal/session-boost-strategy.md | 35 ++++++++++--- pkg/kthena-router/router/router.go | 51 +++++++++++++++++++ pkg/kthena-router/router/router_test.go | 25 +++++++++ 9 files changed, 180 insertions(+), 26 deletions(-) diff --git a/charts/kthena/charts/networking/README.md b/charts/kthena/charts/networking/README.md index c7a7a0328..88307c6c0 100644 --- a/charts/kthena/charts/networking/README.md +++ b/charts/kthena/charts/networking/README.md @@ -52,18 +52,20 @@ kthenaRouter: #### Configuration Parameters -| Parameter | Type | Default | Description | -| ------------------------------------------ | ------- | ---------------- | ------------------------------------------------------------------ | -| `kthenaRouter.fairness.enabled` | boolean | `false` | Enable user-fairness scheduling (mutually exclusive with boost) | -| `kthenaRouter.fairness.windowSize` | string | `"1h"` | Fairness: sliding window duration (1m-1h) | -| `kthenaRouter.fairness.inputTokenWeight` | float | `1.0` | Fairness: weight for input tokens (≥0) | -| `kthenaRouter.fairness.outputTokenWeight` | float | `2.0` | Fairness: weight for output tokens (≥0) | -| `kthenaRouter.fairness.maxConcurrent` | int | `0` | Fairness: global inflight limit (`0` falls back to QPS mode) | -| `kthenaRouter.sessionBoost.enabled` | boolean | `false` | Enable session-boost scheduling (mutually exclusive with fairness) | -| `kthenaRouter.sessionBoost.header` | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions | -| `kthenaRouter.sessionBoost.maxSessions` | int | `4096` | Max recently-completed sessions kept warm (LRU-evicted) | -| `kthenaRouter.sessionBoost.inflightPerPod` | int | `16` | Inflight requests per backend pod; total = perPod x pod count | -| `kthenaRouter.sessionBoost.gracePeriod` | string | `"0s"` | Wait time for a same-session follow-up (disabled by default) | +| Parameter | Type | Default | Description | +| --------------------------------------------- | ------- | ---------------- | ------------------------------------------------------------------ | +| `kthenaRouter.fairness.enabled` | boolean | `false` | Enable user-fairness scheduling (mutually exclusive with boost) | +| `kthenaRouter.fairness.windowSize` | string | `"1h"` | Fairness: sliding window duration (1m-1h) | +| `kthenaRouter.fairness.inputTokenWeight` | float | `1.0` | Fairness: weight for input tokens (≥0) | +| `kthenaRouter.fairness.outputTokenWeight` | float | `2.0` | Fairness: weight for output tokens (≥0) | +| `kthenaRouter.fairness.maxConcurrent` | int | `0` | Fairness: global inflight limit (`0` falls back to QPS mode) | +| `kthenaRouter.sessionBoost.enabled` | boolean | `false` | Enable session-boost scheduling (mutually exclusive with fairness) | +| `kthenaRouter.sessionBoost.header` | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions | +| `kthenaRouter.sessionBoost.maxSessions` | int | `4096` | Max recently-completed sessions kept warm (LRU-evicted) | +| `kthenaRouter.sessionBoost.inflightPerPod` | int | `16` | Inflight requests per backend pod; total = perPod x pod count | +| `kthenaRouter.sessionBoost.gracePeriod` | string | `"0s"` | Wait time for a same-session follow-up (disabled by default) | +| `kthenaRouter.sessionBoost.waitRejectEnabled` | boolean | `false` | Reject requests waiting longer than `maxWait` with HTTP 429 | +| `kthenaRouter.sessionBoost.maxWait` | string | `"30s"` | Max queue wait before 429 (only when `waitRejectEnabled: true`) | #### Session Boost Configuration diff --git a/charts/kthena/charts/networking/templates/kthena-router/component/deployment.yaml b/charts/kthena/charts/networking/templates/kthena-router/component/deployment.yaml index 58f345282..b21d4c547 100644 --- a/charts/kthena/charts/networking/templates/kthena-router/component/deployment.yaml +++ b/charts/kthena/charts/networking/templates/kthena-router/component/deployment.yaml @@ -106,6 +106,12 @@ spec: {{- end }} - name: SESSION_BOOST_GRACE_PERIOD value: {{ .gracePeriod | quote }} + {{- if .waitRejectEnabled }} + - name: SESSION_BOOST_WAIT_REJECT_ENABLED + value: "true" + - name: SESSION_BOOST_MAX_WAIT + value: {{ .maxWait | quote }} + {{- end }} {{- end }} {{- end }} # Access log configuration diff --git a/charts/kthena/charts/networking/values.yaml b/charts/kthena/charts/networking/values.yaml index 54b158891..eaa7b7406 100644 --- a/charts/kthena/charts/networking/values.yaml +++ b/charts/kthena/charts/networking/values.yaml @@ -70,6 +70,14 @@ kthenaRouter: # follow-up to arrive. Disabled by default (0s); enable only if you understand # the latency trade-off for non-boosted requests. gracePeriod: "0s" + # waitRejectEnabled controls whether requests that wait in the session-boost + # queue longer than maxWait are rejected with HTTP 429 instead of continuing to + # wait for the general queue timeout. Disabled by default. + waitRejectEnabled: false + # maxWait is the maximum time a request may wait in the session-boost queue + # before it is rejected with HTTP 429. Only effective when waitRejectEnabled is + # true. Keep it below the general queue timeout so 429 fires before a 504. + maxWait: "30s" # accessLog configuration for request logging accessLog: # enabled controls whether access logging is active diff --git a/charts/kthena/values.yaml b/charts/kthena/values.yaml index 8943e3737..9fa824200 100644 --- a/charts/kthena/values.yaml +++ b/charts/kthena/values.yaml @@ -100,6 +100,14 @@ networking: # -- Wait time after a request completes for a same-session follow-up.
# Disabled by default (`0s`). gracePeriod: "0s" + # -- Reject requests that wait longer than `maxWait` in the session-boost
+ # queue with HTTP 429 instead of waiting for the general queue timeout.
+ # Disabled by default. + waitRejectEnabled: false + # -- Maximum time a request may wait in the session-boost queue before it is
+ # rejected with HTTP 429. Only effective when `waitRejectEnabled` is `true`.
+ # Keep it below the general queue timeout so 429 fires before a 504. + maxWait: "30s" gatewayAPI: # -- Enable Gateway API related features. enabled: false diff --git a/docs/kthena/docs/reference/helm-chart-values.md b/docs/kthena/docs/reference/helm-chart-values.md index 3c9c86a7b..9d567b0b7 100644 --- a/docs/kthena/docs/reference/helm-chart-values.md +++ b/docs/kthena/docs/reference/helm-chart-values.md @@ -37,6 +37,8 @@ A Helm chart for deploying Kthena | networking.kthenaRouter.sessionBoost.header | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions. | | networking.kthenaRouter.sessionBoost.inflightPerPod | int | `16` | Maximum inflight requests admitted per backend pod.
Total inflight limit is this value times the number of backend pods.
`0` or unset uses the router default (16). | | networking.kthenaRouter.sessionBoost.maxSessions | int | `4096` | Maximum number of recently-completed sessions kept warm for boosting.
Bounds an LRU cache; the least-recently-used session is evicted automatically.
Size it by the number of concurrent conversations to keep boosted. | +| networking.kthenaRouter.sessionBoost.maxWait | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 429. Only effective when `waitRejectEnabled` is `true`.
Keep it below the general queue timeout so 429 fires before a 504. | +| networking.kthenaRouter.sessionBoost.waitRejectEnabled | bool | `false` | Reject requests that wait longer than `maxWait` in the session-boost
queue with HTTP 429 instead of waiting for the general queue timeout.
Disabled by default. | | networking.kthenaRouter.terminationGracePeriodSeconds | int | `330` | The router will drain all in-flight requests before forcefully closing connections. | | networking.kthenaRouter.tls.dnsName | string | `"your-domain.com"` | DNS name to use for the certificate. | | networking.kthenaRouter.tls.enabled | bool | `false` | Enable TLS for Kthena Router server. | diff --git a/docs/kthena/docs/user-guide/session-boost.md b/docs/kthena/docs/user-guide/session-boost.md index 60e717069..071f94a8f 100644 --- a/docs/kthena/docs/user-guide/session-boost.md +++ b/docs/kthena/docs/user-guide/session-boost.md @@ -30,7 +30,9 @@ flowchart TD E --> F F -- Yes --> G[Dispatch to backend] F -- No --> H[Wait / backpressure] - H --> F + H --> K{Waited longer than
SESSION_BOOST_MAX_WAIT?} + K -- Yes, wait-reject enabled --> R[Reject with HTTP 429] + K -- No --> F G --> I[Request completes] I --> J[Mark session completed
promote in LRU cache] J --> M[Dequeue next request] @@ -73,6 +75,8 @@ networking: header: "X-Session-ID" maxSessions: 4096 # LRU cache of recently-completed sessions kept warm inflightPerPod: 16 # total inflight = inflightPerPod x backend pod count + waitRejectEnabled: true # reject requests that wait too long with HTTP 429 + maxWait: "30s" # max queue wait before a 429 is returned ``` Apply with Helm: @@ -98,16 +102,22 @@ env: value: "4096" - name: SESSION_BOOST_INFLIGHT_PER_POD value: "16" # total inflight = perPod x backend pod count +- name: SESSION_BOOST_WAIT_REJECT_ENABLED + value: "true" # reject requests that wait too long with HTTP 429 +- name: SESSION_BOOST_MAX_WAIT + value: "30s" # max queue wait before a 429 is returned ``` ## Configuration Reference -| Environment Variable | Purpose | Default | Notes | -| -------------------------------- | -------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `ENABLE_SESSION_BOOST` | Enable the session-boost scheduling strategy | `false` | Mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING` | -| `SESSION_BOOST_HEADER` | HTTP header used to identify conversation sessions | `X-Session-ID` | Must match what your clients send | -| `SESSION_BOOST_MAX_SESSIONS` | Max number of recently-completed sessions kept "warm" for boosting (LRU-bounded) | `4096` | When the cache is full, the least-recently-used session is evicted automatically. Size it by the number of concurrent conversations you want to keep boosted—no time-based tuning required | -| `SESSION_BOOST_INFLIGHT_PER_POD` | Inflight requests admitted per backend pod | `16` | The total inflight limit is this value multiplied by the number of backend pods. Size it from the per-pod concurrency (e.g., vLLM's `--max-num-seqs`) | +| Environment Variable | Purpose | Default | Notes | +| ----------------------------------- | -------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ENABLE_SESSION_BOOST` | Enable the session-boost scheduling strategy | `false` | Mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING` | +| `SESSION_BOOST_HEADER` | HTTP header used to identify conversation sessions | `X-Session-ID` | Must match what your clients send | +| `SESSION_BOOST_MAX_SESSIONS` | Max number of recently-completed sessions kept "warm" for boosting (LRU-bounded) | `4096` | When the cache is full, the least-recently-used session is evicted automatically. Size it by the number of concurrent conversations you want to keep boosted—no time-based tuning required | +| `SESSION_BOOST_INFLIGHT_PER_POD` | Inflight requests admitted per backend pod | `16` | The total inflight limit is this value multiplied by the number of backend pods. Size it from the per-pod concurrency (e.g., vLLM's `--max-num-seqs`) | +| `SESSION_BOOST_WAIT_REJECT_ENABLED` | Reject requests that wait too long in the queue with HTTP 429 | `false` | When `true`, a request queued longer than `SESSION_BOOST_MAX_WAIT` is rejected with `429 Too Many Requests` instead of continuing to wait | +| `SESSION_BOOST_MAX_WAIT` | Maximum time a request may wait in the queue before it is rejected with 429 | `30s` | Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. Keep it below `FAIRNESS_QUEUE_TIMEOUT` so the 429 fires before the general queue timeout (504) | > `SESSION_BOOST_GRACE_PERIOD` is intentionally omitted from the table above. It is an advanced, scenario-specific knob that is disabled by default; see [Advanced: Grace Period](#advanced-grace-period-use-with-caution). @@ -174,6 +184,19 @@ The queue uses two-level admission control to avoid flooding backends: When a request completes, the queue immediately attempts to dequeue the next request (release-driven dequeue) rather than waiting for the next metrics refresh. +### Queue Wait Timeout (429 Rejection) + +Under heavy load a request may sit in the queue for a long time before backend capacity frees up. By default the router waits until the general queue timeout (`FAIRNESS_QUEUE_TIMEOUT`) expires and then returns `504 Gateway Timeout`. For latency-sensitive front ends it is often better to **fail fast** and let the client retry or shed load. + +Session boost provides an optional **wait timeout** that rejects over-queued requests early with `429 Too Many Requests`: + +- Enable it with `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. +- Set the threshold with `SESSION_BOOST_MAX_WAIT` (default `30s`). + +When enabled, if a request waits in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router removes it from the queue and responds with `429 Too Many Requests`. This is distinct from the general queue timeout: the 429 signals *backpressure* (the queue is saturated and the client should back off or retry), whereas the `504` from `FAIRNESS_QUEUE_TIMEOUT` signals a general processing timeout. + +> Keep `SESSION_BOOST_MAX_WAIT` **below** `FAIRNESS_QUEUE_TIMEOUT`. If it is larger, the general queue timeout (504) fires first and the 429 rejection never takes effect. The feature only applies when session boost is enabled; it has no effect in user-fairness mode. + ## Session Boost vs User Fairness Session boost and user fairness are two mutually exclusive scheduling strategies for the per-model request queue; they configure that queue for different goals: @@ -249,6 +272,14 @@ Session boost only controls queue ordering. If the boosted request is routed to Each tracked session consumes minimal memory (just a session ID). The tracker is a bounded LRU cache, so total memory is capped at `SESSION_BOOST_MAX_SESSIONS` entries regardless of how much traffic flows through the router. If memory is a concern, reduce `SESSION_BOOST_MAX_SESSIONS`. +### Clients receive HTTP 429 responses + +When `SESSION_BOOST_WAIT_REJECT_ENABLED=true`, the router returns `429 Too Many Requests` for requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT`. This is expected backpressure behavior under sustained overload—it means the backends cannot keep up with demand. If you see more 429s than desired: + +1. Confirm the backends are healthy and scaled appropriately; add capacity if they are saturated. +2. Increase `SESSION_BOOST_MAX_WAIT` to allow requests to wait longer before rejection (but keep it below `FAIRNESS_QUEUE_TIMEOUT`). +3. Disable the feature (`SESSION_BOOST_WAIT_REJECT_ENABLED=false`) to fall back to the general queue timeout (504) behavior. + ## Advanced: Grace Period (Use With Caution) :::warning diff --git a/docs/proposal/session-boost-strategy.md b/docs/proposal/session-boost-strategy.md index 159b83c16..aa57d9d10 100644 --- a/docs/proposal/session-boost-strategy.md +++ b/docs/proposal/session-boost-strategy.md @@ -39,6 +39,7 @@ Enabling session boost therefore reconfigures the same per-model priority queue 3. **KV cache optimization**: Prioritize follow-up requests from recently completed sessions to maximize warm cache hits. 4. **Grace period (advanced, off by default)**: An optional, tricky tuning knob that, after a request completes, briefly holds the dequeue slot for a potential follow-up from the same session before dispatching unrelated requests. It is **disabled by default** and should only be enabled by operators who fully understand that it deliberately delays unrelated requests in exchange for a higher same-session prefix-cache hit rate. 5. **Backpressure-aware**: Respect backend pod capacity to avoid flooding, using two-level admission control (inflight limit + backend metrics). +6. **Fail-fast queue wait timeout (optional, off by default)**: Optionally reject requests that wait in the queue longer than a configurable threshold with HTTP 429, so latency-sensitive clients shed load or retry instead of blocking until the general queue timeout returns a 504. #### Non-Goals @@ -218,13 +219,15 @@ The net effect is a strict precedence: **grace timing → inflight gate → back #### Configuration -| Environment Variable | Default | Description | -| -------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ENABLE_SESSION_BOOST` | `false` | Enable the session-boost scheduling strategy (mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING`) | -| `SESSION_BOOST_HEADER` | `X-Session-ID` | HTTP header used to identify conversation sessions | -| `SESSION_BOOST_MAX_SESSIONS` | `4096` | Maximum number of recently-completed sessions kept warm for boosting. Bounds an LRU cache; the least-recently-used session is evicted when exceeded. Sized by session count, not time | -| `SESSION_BOOST_GRACE_PERIOD` | `0` | Wait time after release for same-session follow-up. Disabled by default; enable only when you understand the latency trade-off | -| `SESSION_BOOST_INFLIGHT_PER_POD` | `16` | Inflight requests admitted per backend pod; total inflight = perPod x backend pod count. Size it from the estimated per-pod concurrency (e.g. vLLM's --max-num-seqs) | +| Environment Variable | Default | Description | +| ----------------------------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ENABLE_SESSION_BOOST` | `false` | Enable the session-boost scheduling strategy (mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING`) | +| `SESSION_BOOST_HEADER` | `X-Session-ID` | HTTP header used to identify conversation sessions | +| `SESSION_BOOST_MAX_SESSIONS` | `4096` | Maximum number of recently-completed sessions kept warm for boosting. Bounds an LRU cache; the least-recently-used session is evicted when exceeded. Sized by session count, not time | +| `SESSION_BOOST_GRACE_PERIOD` | `0` | Wait time after release for same-session follow-up. Disabled by default; enable only when you understand the latency trade-off | +| `SESSION_BOOST_INFLIGHT_PER_POD` | `16` | Inflight requests admitted per backend pod; total inflight = perPod x backend pod count. Size it from the estimated per-pod concurrency (e.g. vLLM's --max-num-seqs) | +| `SESSION_BOOST_WAIT_REJECT_ENABLED` | `false` | Reject requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT` with HTTP 429 instead of waiting for the general queue timeout | +| `SESSION_BOOST_MAX_WAIT` | `30s` | Maximum time a request may wait in the queue before it is rejected with HTTP 429. Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`; keep it below `FAIRNESS_QUEUE_TIMEOUT` | ### Design Details @@ -297,6 +300,24 @@ The queue uses two-level admission control: When a request completes (Release), the queue immediately attempts to dequeue the next request (release-driven dequeue), ensuring minimal latency between sequential requests. The loop is fully event-driven — there is no independent polling timer. In single-router operation every moment a backend frees capacity coincides with one of our own requests completing (a release), so release and arrival events alone cover every dequeue opportunity; the capacity check simply reads the pod metrics already scraped by the store (`METRICS_SCRAPE_INTERVAL`). +#### Queue Wait Timeout (429 Rejection) + +Under sustained overload the two-level admission control legitimately holds requests in the queue until backend capacity frees up. By default a queued request waits until the router's general queue timeout (`FAIRNESS_QUEUE_TIMEOUT`) expires, at which point the request context is cancelled and the client receives `504 Gateway Timeout`. For latency-sensitive front ends this coarse timeout can be undesirable: the client would rather be told *quickly* that the system is saturated so it can retry elsewhere or shed load. + +The optional queue wait timeout provides this fail-fast behavior: + +- It is enabled with `SESSION_BOOST_WAIT_REJECT_ENABLED=true` and disabled by default. +- The threshold is configured with `SESSION_BOOST_MAX_WAIT` (default `30s`). +- When a request has been waiting in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router stops waiting, removes the request from the queue, and responds with `429 Too Many Requests`. + +It is implemented in the router's request-handling path rather than in the queue's dequeue loop, mirroring how the existing `FAIRNESS_QUEUE_TIMEOUT` (504) is handled. When session boost is enabled and the feature is on, the handler arms a timer for `SESSION_BOOST_MAX_WAIT` alongside the admission (`NotifyChan`) and cancellation (`reqCtx.Done()`) signals it already waits on: + +- **Admitted first**: the request proceeds to the backend as usual (no rejection). +- **Wait timer fires first**: the handler cancels the request context — which makes the queue drop the request from the heap via its existing cancellation check (`isCancelled` / `drainCancelledLocked`), reusing the same cleanup path as client disconnects and the general timeout — releases any permit that may have been granted concurrently, and returns `429 Too Many Requests`. +- **General timeout / client disconnect fires first**: the existing behavior is unchanged (504 or 503 respectively). + +Because it reuses the queue's cancellation cleanup, the wait timeout adds no new state to the queue and no extra polling. It is orthogonal to the inflight and backend capacity gates: those gates decide *whether* a request may run, while the wait timeout only bounds *how long* a request is willing to wait before giving up. Operators should keep `SESSION_BOOST_MAX_WAIT` below `FAIRNESS_QUEUE_TIMEOUT`; otherwise the general 504 timeout fires first and the 429 rejection never takes effect. The feature only applies in session-boost mode. + ### Multi-Turn Conversation Advantages #### 1. Prefix Cache Hit Rate Improvement diff --git a/pkg/kthena-router/router/router.go b/pkg/kthena-router/router/router.go index 469fef76c..386efb9b8 100644 --- a/pkg/kthena-router/router/router.go +++ b/pkg/kthena-router/router/router.go @@ -92,6 +92,13 @@ type Router struct { queueTimeout time.Duration tokenWeight float64 // Weight for token-based priority in the fairness strategy (default 1.0) requestNumWeight float64 // Weight for request-count-based priority in the fairness strategy (default 0.0) + + // Session-boost wait-reject configuration. When sessionBoostWaitRejectEnabled + // is true, a request that waits in the session-boost queue longer than + // sessionBoostMaxWait is rejected with HTTP 429 instead of continuing to wait + // for the general queue timeout. + sessionBoostWaitRejectEnabled bool + sessionBoostMaxWait time.Duration } // ActiveRequestCount returns the number of requests currently being handled by the router. @@ -182,6 +189,9 @@ func NewRouter(store datastore.Store, routerConfigPath string) *Router { queueTimeout: parseQueueTimeout(), tokenWeight: parseEnvFloat("FAIRNESS_PRIORITY_TOKEN_WEIGHT", 1.0), requestNumWeight: parseEnvFloat("FAIRNESS_PRIORITY_REQUEST_NUM_WEIGHT", 0.0), + + sessionBoostWaitRejectEnabled: getEnvBool("SESSION_BOOST_WAIT_REJECT_ENABLED", false), + sessionBoostMaxWait: parseSessionBoostMaxWait(), } } @@ -197,6 +207,24 @@ func parseQueueTimeout() time.Duration { return defaultQueueTimeout } +// defaultSessionBoostMaxWait is the maximum time a request may wait in the +// session-boost queue before it is rejected with HTTP 429, used when +// SESSION_BOOST_MAX_WAIT is not set or invalid and wait-reject is enabled. +const defaultSessionBoostMaxWait = 30 * time.Second + +// parseSessionBoostMaxWait reads the session-boost max queue wait from the +// SESSION_BOOST_MAX_WAIT environment variable. It only takes effect when +// SESSION_BOOST_WAIT_REJECT_ENABLED is true. +func parseSessionBoostMaxWait() time.Duration { + if s, ok := os.LookupEnv("SESSION_BOOST_MAX_WAIT"); ok { + if d, err := time.ParseDuration(s); err == nil && d > 0 { + return d + } + klog.Warningf("Invalid SESSION_BOOST_MAX_WAIT %q, using default %v", s, defaultSessionBoostMaxWait) + } + return defaultSessionBoostMaxWait +} + func parseEnvFloat(key string, fallback float64) float64 { if s, ok := os.LookupEnv(key); ok { if v, err := strconv.ParseFloat(s, 64); err == nil && !math.IsNaN(v) && !math.IsInf(v, 0) && v >= 0 { @@ -1113,6 +1141,17 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ return fmt.Errorf("failed to enqueue request: %v", err) } + // Session-boost wait-reject: when enabled, a request that waits in the queue + // longer than sessionBoostMaxWait is rejected with HTTP 429 instead of waiting + // out the general queue timeout (which returns 504). The timer only arms in + // session-boost mode when the feature is enabled and a positive max wait is set. + var waitRejectCh <-chan time.Time + if EnableSessionBoost && r.sessionBoostWaitRejectEnabled && r.sessionBoostMaxWait > 0 { + waitRejectTimer := time.NewTimer(r.sessionBoostMaxWait) + defer waitRejectTimer.Stop() + waitRejectCh = waitRejectTimer.C + } + select { case <-queueReq.NotifyChan: if queueReq.Release != nil { @@ -1129,6 +1168,18 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ r.store.MarkSessionRequestCompleted(modelName, sessionID) } return nil + case <-waitRejectCh: + // Exceeded the session-boost maximum queue wait. Cancel the request context + // so the queue drops it from the heap (via its cancellation check), release + // any permit if one was concurrently granted, and reject the client with 429. + cancel() + if queueReq.Release != nil { + queueReq.Release() + } + klog.Warningf("[SessionBoost] request rejected after exceeding max queue wait: reqID=%s sessionID=%s user=%s model=%s maxWait=%v", + requestID, sessionID, userId, modelName, r.sessionBoostMaxWait) + c.AbortWithStatusJSON(http.StatusTooManyRequests, "Request rejected: exceeded maximum session boost queue wait time") + return fmt.Errorf("request rejected: exceeded session boost max queue wait") case <-reqCtx.Done(): if queueReq.Release != nil { queueReq.Release() diff --git a/pkg/kthena-router/router/router_test.go b/pkg/kthena-router/router/router_test.go index 60a84cd57..9af0f4a4e 100644 --- a/pkg/kthena-router/router/router_test.go +++ b/pkg/kthena-router/router/router_test.go @@ -1363,6 +1363,10 @@ func TestHandleFairnessScheduling(t *testing.T) { wantErrMsg string wantHTTPStatus int wantBodyContains string + // Session-boost wait-reject configuration for the test case. + enableSessionBoost bool + waitRejectEnabled bool + sessionBoostMaxWait time.Duration }{ { name: "happy path with userId", @@ -1410,6 +1414,18 @@ func TestHandleFairnessScheduling(t *testing.T) { wantErrMsg: "failed to enqueue request", wantHTTPStatus: http.StatusInternalServerError, }, + { + name: "session boost wait-reject returns 429", + fairnessTimeout: 10 * time.Second, + setUserID: true, + storeWrapper: func(real datastore.Store) datastore.Store { return &blockingEnqueueStore{Store: real} }, + enableSessionBoost: true, + waitRejectEnabled: true, + sessionBoostMaxWait: 50 * time.Millisecond, + wantErr: true, + wantErrMsg: "exceeded session boost max queue wait", + wantHTTPStatus: http.StatusTooManyRequests, + }, } for _, tt := range tests { @@ -1418,6 +1434,15 @@ func TestHandleFairnessScheduling(t *testing.T) { defer backend.Close() router.queueTimeout = tt.fairnessTimeout + router.sessionBoostWaitRejectEnabled = tt.waitRejectEnabled + if tt.sessionBoostMaxWait > 0 { + router.sessionBoostMaxWait = tt.sessionBoostMaxWait + } + if tt.enableSessionBoost { + prev := EnableSessionBoost + EnableSessionBoost = true + defer func() { EnableSessionBoost = prev }() + } if tt.storeWrapper != nil { router.store = tt.storeWrapper(store) } From 3289d93d7fa99793039930e5e8475c7a0b7c4076 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Wed, 1 Jul 2026 06:41:22 +0000 Subject: [PATCH 02/11] session boost: decouple queue wait timeout from FAIRNESS_QUEUE_TIMEOUT FAIRNESS_QUEUE_TIMEOUT now applies only to the user-fairness queue. In session-boost mode the request context no longer carries the fairness deadline; the only server-side queue-wait bound is the optional SESSION_BOOST_MAX_WAIT (429). Docs updated accordingly. Signed-off-by: YaoZengzeng Signed-off-by: YaoZengzeng --- .../docs/user-guide/fairness-scheduling.md | 2 +- docs/kthena/docs/user-guide/session-boost.md | 14 +++++++------- docs/proposal/session-boost-strategy.md | 16 ++++++++-------- pkg/kthena-router/router/router.go | 16 ++++++++++++++-- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/kthena/docs/user-guide/fairness-scheduling.md b/docs/kthena/docs/user-guide/fairness-scheduling.md index 73ce21f76..1aaa1d33a 100644 --- a/docs/kthena/docs/user-guide/fairness-scheduling.md +++ b/docs/kthena/docs/user-guide/fairness-scheduling.md @@ -127,7 +127,7 @@ env: | `FAIRNESS_WINDOW_SIZE` | Sliding window used to track recent usage | runtime default `5m` | The Helm chart default sets this to `1h` when fairness is enabled | | `FAIRNESS_INPUT_TOKEN_WEIGHT` | Weight applied to input tokens when recording usage | `1.0` | Used by the token tracker | | `FAIRNESS_OUTPUT_TOKEN_WEIGHT` | Weight applied to output tokens when recording usage | `2.0` | Used by the token tracker | -| `FAIRNESS_QUEUE_TIMEOUT` | Maximum time a request may wait in the fairness queue | `60s` | Waiting longer returns a timeout to the client | +| `FAIRNESS_QUEUE_TIMEOUT` | Maximum time a request may wait in the fairness queue | `60s` | Waiting longer returns a timeout (504) to the client. Applies only to the user-fairness queue, not session boost | ### Queue Policy Settings diff --git a/docs/kthena/docs/user-guide/session-boost.md b/docs/kthena/docs/user-guide/session-boost.md index 071f94a8f..5a01067b9 100644 --- a/docs/kthena/docs/user-guide/session-boost.md +++ b/docs/kthena/docs/user-guide/session-boost.md @@ -117,7 +117,7 @@ env: | `SESSION_BOOST_MAX_SESSIONS` | Max number of recently-completed sessions kept "warm" for boosting (LRU-bounded) | `4096` | When the cache is full, the least-recently-used session is evicted automatically. Size it by the number of concurrent conversations you want to keep boosted—no time-based tuning required | | `SESSION_BOOST_INFLIGHT_PER_POD` | Inflight requests admitted per backend pod | `16` | The total inflight limit is this value multiplied by the number of backend pods. Size it from the per-pod concurrency (e.g., vLLM's `--max-num-seqs`) | | `SESSION_BOOST_WAIT_REJECT_ENABLED` | Reject requests that wait too long in the queue with HTTP 429 | `false` | When `true`, a request queued longer than `SESSION_BOOST_MAX_WAIT` is rejected with `429 Too Many Requests` instead of continuing to wait | -| `SESSION_BOOST_MAX_WAIT` | Maximum time a request may wait in the queue before it is rejected with 429 | `30s` | Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. Keep it below `FAIRNESS_QUEUE_TIMEOUT` so the 429 fires before the general queue timeout (504) | +| `SESSION_BOOST_MAX_WAIT` | Maximum time a request may wait in the queue before it is rejected with 429 | `30s` | Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. This is the only server-side queue-wait bound in session-boost mode; `FAIRNESS_QUEUE_TIMEOUT` does not apply | > `SESSION_BOOST_GRACE_PERIOD` is intentionally omitted from the table above. It is an advanced, scenario-specific knob that is disabled by default; see [Advanced: Grace Period](#advanced-grace-period-use-with-caution). @@ -186,16 +186,16 @@ When a request completes, the queue immediately attempts to dequeue the next req ### Queue Wait Timeout (429 Rejection) -Under heavy load a request may sit in the queue for a long time before backend capacity frees up. By default the router waits until the general queue timeout (`FAIRNESS_QUEUE_TIMEOUT`) expires and then returns `504 Gateway Timeout`. For latency-sensitive front ends it is often better to **fail fast** and let the client retry or shed load. +Under heavy load a request may sit in the queue for a long time before backend capacity frees up. Session boost does **not** apply the fairness queue timeout (`FAIRNESS_QUEUE_TIMEOUT` only governs the user-fairness queue); by default a queued request in session-boost mode waits until backend capacity frees up or the client disconnects. For latency-sensitive front ends it is often better to **fail fast** and let the client retry or shed load. -Session boost provides an optional **wait timeout** that rejects over-queued requests early with `429 Too Many Requests`: +Session boost provides its own optional **wait timeout** that rejects over-queued requests early with `429 Too Many Requests`: - Enable it with `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. - Set the threshold with `SESSION_BOOST_MAX_WAIT` (default `30s`). -When enabled, if a request waits in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router removes it from the queue and responds with `429 Too Many Requests`. This is distinct from the general queue timeout: the 429 signals *backpressure* (the queue is saturated and the client should back off or retry), whereas the `504` from `FAIRNESS_QUEUE_TIMEOUT` signals a general processing timeout. +When enabled, if a request waits in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router removes it from the queue and responds with `429 Too Many Requests`. The 429 signals *backpressure* (the queue is saturated and the client should back off or retry). -> Keep `SESSION_BOOST_MAX_WAIT` **below** `FAIRNESS_QUEUE_TIMEOUT`. If it is larger, the general queue timeout (504) fires first and the 429 rejection never takes effect. The feature only applies when session boost is enabled; it has no effect in user-fairness mode. +> `SESSION_BOOST_MAX_WAIT` is the only server-side queue-wait bound in session-boost mode. `FAIRNESS_QUEUE_TIMEOUT` has no effect here—it applies exclusively to the user-fairness strategy. Without `SESSION_BOOST_WAIT_REJECT_ENABLED`, a session-boost request is bounded only by client disconnect. ## Session Boost vs User Fairness @@ -277,8 +277,8 @@ Each tracked session consumes minimal memory (just a session ID). The tracker is When `SESSION_BOOST_WAIT_REJECT_ENABLED=true`, the router returns `429 Too Many Requests` for requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT`. This is expected backpressure behavior under sustained overload—it means the backends cannot keep up with demand. If you see more 429s than desired: 1. Confirm the backends are healthy and scaled appropriately; add capacity if they are saturated. -2. Increase `SESSION_BOOST_MAX_WAIT` to allow requests to wait longer before rejection (but keep it below `FAIRNESS_QUEUE_TIMEOUT`). -3. Disable the feature (`SESSION_BOOST_WAIT_REJECT_ENABLED=false`) to fall back to the general queue timeout (504) behavior. +2. Increase `SESSION_BOOST_MAX_WAIT` to allow requests to wait longer before rejection. +3. Disable the feature (`SESSION_BOOST_WAIT_REJECT_ENABLED=false`) so requests are no longer rejected on a wait timeout (they then wait until backend capacity frees up or the client disconnects). ## Advanced: Grace Period (Use With Caution) diff --git a/docs/proposal/session-boost-strategy.md b/docs/proposal/session-boost-strategy.md index aa57d9d10..2a546e0e8 100644 --- a/docs/proposal/session-boost-strategy.md +++ b/docs/proposal/session-boost-strategy.md @@ -39,7 +39,7 @@ Enabling session boost therefore reconfigures the same per-model priority queue 3. **KV cache optimization**: Prioritize follow-up requests from recently completed sessions to maximize warm cache hits. 4. **Grace period (advanced, off by default)**: An optional, tricky tuning knob that, after a request completes, briefly holds the dequeue slot for a potential follow-up from the same session before dispatching unrelated requests. It is **disabled by default** and should only be enabled by operators who fully understand that it deliberately delays unrelated requests in exchange for a higher same-session prefix-cache hit rate. 5. **Backpressure-aware**: Respect backend pod capacity to avoid flooding, using two-level admission control (inflight limit + backend metrics). -6. **Fail-fast queue wait timeout (optional, off by default)**: Optionally reject requests that wait in the queue longer than a configurable threshold with HTTP 429, so latency-sensitive clients shed load or retry instead of blocking until the general queue timeout returns a 504. +6. **Fail-fast queue wait timeout (optional, off by default)**: Optionally reject requests that wait in the queue longer than a configurable threshold with HTTP 429, so latency-sensitive clients shed load or retry instead of waiting indefinitely for backend capacity. `FAIRNESS_QUEUE_TIMEOUT` governs only the user-fairness queue and does not apply in session-boost mode. #### Non-Goals @@ -226,8 +226,8 @@ The net effect is a strict precedence: **grace timing → inflight gate → back | `SESSION_BOOST_MAX_SESSIONS` | `4096` | Maximum number of recently-completed sessions kept warm for boosting. Bounds an LRU cache; the least-recently-used session is evicted when exceeded. Sized by session count, not time | | `SESSION_BOOST_GRACE_PERIOD` | `0` | Wait time after release for same-session follow-up. Disabled by default; enable only when you understand the latency trade-off | | `SESSION_BOOST_INFLIGHT_PER_POD` | `16` | Inflight requests admitted per backend pod; total inflight = perPod x backend pod count. Size it from the estimated per-pod concurrency (e.g. vLLM's --max-num-seqs) | -| `SESSION_BOOST_WAIT_REJECT_ENABLED` | `false` | Reject requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT` with HTTP 429 instead of waiting for the general queue timeout | -| `SESSION_BOOST_MAX_WAIT` | `30s` | Maximum time a request may wait in the queue before it is rejected with HTTP 429. Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`; keep it below `FAIRNESS_QUEUE_TIMEOUT` | +| `SESSION_BOOST_WAIT_REJECT_ENABLED` | `false` | Reject requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT` with HTTP 429. `FAIRNESS_QUEUE_TIMEOUT` does not apply in session-boost mode | +| `SESSION_BOOST_MAX_WAIT` | `30s` | Maximum time a request may wait in the queue before it is rejected with HTTP 429. Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`; the only server-side queue-wait bound in session-boost mode | ### Design Details @@ -302,7 +302,7 @@ When a request completes (Release), the queue immediately attempts to dequeue th #### Queue Wait Timeout (429 Rejection) -Under sustained overload the two-level admission control legitimately holds requests in the queue until backend capacity frees up. By default a queued request waits until the router's general queue timeout (`FAIRNESS_QUEUE_TIMEOUT`) expires, at which point the request context is cancelled and the client receives `504 Gateway Timeout`. For latency-sensitive front ends this coarse timeout can be undesirable: the client would rather be told *quickly* that the system is saturated so it can retry elsewhere or shed load. +Under sustained overload the two-level admission control legitimately holds requests in the queue until backend capacity frees up. `FAIRNESS_QUEUE_TIMEOUT` governs **only** the user-fairness queue and does not apply in session-boost mode, so by default a queued session-boost request waits until backend capacity frees up or the client disconnects (which returns `503`). For latency-sensitive front ends this open-ended wait can be undesirable: the client would rather be told *quickly* that the system is saturated so it can retry elsewhere or shed load. The optional queue wait timeout provides this fail-fast behavior: @@ -310,13 +310,13 @@ The optional queue wait timeout provides this fail-fast behavior: - The threshold is configured with `SESSION_BOOST_MAX_WAIT` (default `30s`). - When a request has been waiting in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router stops waiting, removes the request from the queue, and responds with `429 Too Many Requests`. -It is implemented in the router's request-handling path rather than in the queue's dequeue loop, mirroring how the existing `FAIRNESS_QUEUE_TIMEOUT` (504) is handled. When session boost is enabled and the feature is on, the handler arms a timer for `SESSION_BOOST_MAX_WAIT` alongside the admission (`NotifyChan`) and cancellation (`reqCtx.Done()`) signals it already waits on: +It is implemented in the router's request-handling path rather than in the queue's dequeue loop, mirroring how the fairness queue's own `FAIRNESS_QUEUE_TIMEOUT` (504) is handled. In session-boost mode the request context is created **without** the fairness deadline, so `FAIRNESS_QUEUE_TIMEOUT` has no effect. When the feature is on, the handler arms a timer for `SESSION_BOOST_MAX_WAIT` alongside the admission (`NotifyChan`) and cancellation (`reqCtx.Done()`) signals it already waits on: - **Admitted first**: the request proceeds to the backend as usual (no rejection). -- **Wait timer fires first**: the handler cancels the request context — which makes the queue drop the request from the heap via its existing cancellation check (`isCancelled` / `drainCancelledLocked`), reusing the same cleanup path as client disconnects and the general timeout — releases any permit that may have been granted concurrently, and returns `429 Too Many Requests`. -- **General timeout / client disconnect fires first**: the existing behavior is unchanged (504 or 503 respectively). +- **Wait timer fires first**: the handler cancels the request context — which makes the queue drop the request from the heap via its existing cancellation check (`isCancelled` / `drainCancelledLocked`), reusing the same cleanup path as client disconnects — releases any permit that may have been granted concurrently, and returns `429 Too Many Requests`. +- **Client disconnect fires first**: the existing behavior is unchanged (`503`). -Because it reuses the queue's cancellation cleanup, the wait timeout adds no new state to the queue and no extra polling. It is orthogonal to the inflight and backend capacity gates: those gates decide *whether* a request may run, while the wait timeout only bounds *how long* a request is willing to wait before giving up. Operators should keep `SESSION_BOOST_MAX_WAIT` below `FAIRNESS_QUEUE_TIMEOUT`; otherwise the general 504 timeout fires first and the 429 rejection never takes effect. The feature only applies in session-boost mode. +Because it reuses the queue's cancellation cleanup, the wait timeout adds no new state to the queue and no extra polling. It is orthogonal to the inflight and backend capacity gates: those gates decide *whether* a request may run, while the wait timeout only bounds *how long* a request is willing to wait before giving up. `SESSION_BOOST_MAX_WAIT` is the only server-side queue-wait bound in session-boost mode; the feature only applies in session-boost mode. ### Multi-Turn Conversation Advantages diff --git a/pkg/kthena-router/router/router.go b/pkg/kthena-router/router/router.go index 386efb9b8..29a6d0d0a 100644 --- a/pkg/kthena-router/router/router.go +++ b/pkg/kthena-router/router/router.go @@ -1108,8 +1108,20 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ klog.V(4).Infof("[FairnessScheduling] incoming request: reqID=%s user=%s model=%s", requestID, userId, modelName) - // Create request-scoped context that unifies client disconnect and server timeout - reqCtx, cancel := context.WithTimeout(c.Request.Context(), r.queueTimeout) + // Create the request-scoped context that also drives the queue's cancellation + // cleanup (CancelCh). The general queue-wait deadline differs by strategy: + // - Fairness mode: bounded by FAIRNESS_QUEUE_TIMEOUT; exceeding it returns 504. + // - Session-boost mode: FAIRNESS_QUEUE_TIMEOUT does NOT apply. Session boost has + // its own independent wait control via SESSION_BOOST_MAX_WAIT (returns 429 when + // SESSION_BOOST_WAIT_REJECT_ENABLED is set); otherwise the request is bounded + // only by client disconnect. + var reqCtx context.Context + var cancel context.CancelFunc + if EnableSessionBoost { + reqCtx, cancel = context.WithCancel(c.Request.Context()) + } else { + reqCtx, cancel = context.WithTimeout(c.Request.Context(), r.queueTimeout) + } defer cancel() var pri float64 From 729f67917e8bbee98afeedf57849bc8acaf42750 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Wed, 1 Jul 2026 06:45:54 +0000 Subject: [PATCH 03/11] docs: reflow session boost/fairness config tables Realign markdown config tables after the FAIRNESS_QUEUE_TIMEOUT note length change. No content change. Signed-off-by: YaoZengzeng Signed-off-by: YaoZengzeng --- .../docs/user-guide/fairness-scheduling.md | 12 ++++++------ docs/kthena/docs/user-guide/session-boost.md | 2 +- docs/proposal/session-boost-strategy.md | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/kthena/docs/user-guide/fairness-scheduling.md b/docs/kthena/docs/user-guide/fairness-scheduling.md index 1aaa1d33a..1cad45885 100644 --- a/docs/kthena/docs/user-guide/fairness-scheduling.md +++ b/docs/kthena/docs/user-guide/fairness-scheduling.md @@ -121,12 +121,12 @@ env: ### Core Settings -| Environment Variable | Purpose | Default | Notes | -| ------------------------------ | ----------------------------------------------------- | -------------------- | --------------------------------------------------------------------- | -| `ENABLE_FAIRNESS_SCHEDULING` | Enables fairness scheduling in the router | `false` | Global feature switch. Mutually exclusive with `ENABLE_SESSION_BOOST` | -| `FAIRNESS_WINDOW_SIZE` | Sliding window used to track recent usage | runtime default `5m` | The Helm chart default sets this to `1h` when fairness is enabled | -| `FAIRNESS_INPUT_TOKEN_WEIGHT` | Weight applied to input tokens when recording usage | `1.0` | Used by the token tracker | -| `FAIRNESS_OUTPUT_TOKEN_WEIGHT` | Weight applied to output tokens when recording usage | `2.0` | Used by the token tracker | +| Environment Variable | Purpose | Default | Notes | +| ------------------------------ | ----------------------------------------------------- | -------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `ENABLE_FAIRNESS_SCHEDULING` | Enables fairness scheduling in the router | `false` | Global feature switch. Mutually exclusive with `ENABLE_SESSION_BOOST` | +| `FAIRNESS_WINDOW_SIZE` | Sliding window used to track recent usage | runtime default `5m` | The Helm chart default sets this to `1h` when fairness is enabled | +| `FAIRNESS_INPUT_TOKEN_WEIGHT` | Weight applied to input tokens when recording usage | `1.0` | Used by the token tracker | +| `FAIRNESS_OUTPUT_TOKEN_WEIGHT` | Weight applied to output tokens when recording usage | `2.0` | Used by the token tracker | | `FAIRNESS_QUEUE_TIMEOUT` | Maximum time a request may wait in the fairness queue | `60s` | Waiting longer returns a timeout (504) to the client. Applies only to the user-fairness queue, not session boost | ### Queue Policy Settings diff --git a/docs/kthena/docs/user-guide/session-boost.md b/docs/kthena/docs/user-guide/session-boost.md index 5a01067b9..e85c56cc0 100644 --- a/docs/kthena/docs/user-guide/session-boost.md +++ b/docs/kthena/docs/user-guide/session-boost.md @@ -117,7 +117,7 @@ env: | `SESSION_BOOST_MAX_SESSIONS` | Max number of recently-completed sessions kept "warm" for boosting (LRU-bounded) | `4096` | When the cache is full, the least-recently-used session is evicted automatically. Size it by the number of concurrent conversations you want to keep boosted—no time-based tuning required | | `SESSION_BOOST_INFLIGHT_PER_POD` | Inflight requests admitted per backend pod | `16` | The total inflight limit is this value multiplied by the number of backend pods. Size it from the per-pod concurrency (e.g., vLLM's `--max-num-seqs`) | | `SESSION_BOOST_WAIT_REJECT_ENABLED` | Reject requests that wait too long in the queue with HTTP 429 | `false` | When `true`, a request queued longer than `SESSION_BOOST_MAX_WAIT` is rejected with `429 Too Many Requests` instead of continuing to wait | -| `SESSION_BOOST_MAX_WAIT` | Maximum time a request may wait in the queue before it is rejected with 429 | `30s` | Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. This is the only server-side queue-wait bound in session-boost mode; `FAIRNESS_QUEUE_TIMEOUT` does not apply | +| `SESSION_BOOST_MAX_WAIT` | Maximum time a request may wait in the queue before it is rejected with 429 | `30s` | Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. This is the only server-side queue-wait bound in session-boost mode; `FAIRNESS_QUEUE_TIMEOUT` does not apply | > `SESSION_BOOST_GRACE_PERIOD` is intentionally omitted from the table above. It is an advanced, scenario-specific knob that is disabled by default; see [Advanced: Grace Period](#advanced-grace-period-use-with-caution). diff --git a/docs/proposal/session-boost-strategy.md b/docs/proposal/session-boost-strategy.md index 2a546e0e8..1a17103a2 100644 --- a/docs/proposal/session-boost-strategy.md +++ b/docs/proposal/session-boost-strategy.md @@ -219,14 +219,14 @@ The net effect is a strict precedence: **grace timing → inflight gate → back #### Configuration -| Environment Variable | Default | Description | -| ----------------------------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ENABLE_SESSION_BOOST` | `false` | Enable the session-boost scheduling strategy (mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING`) | -| `SESSION_BOOST_HEADER` | `X-Session-ID` | HTTP header used to identify conversation sessions | -| `SESSION_BOOST_MAX_SESSIONS` | `4096` | Maximum number of recently-completed sessions kept warm for boosting. Bounds an LRU cache; the least-recently-used session is evicted when exceeded. Sized by session count, not time | -| `SESSION_BOOST_GRACE_PERIOD` | `0` | Wait time after release for same-session follow-up. Disabled by default; enable only when you understand the latency trade-off | -| `SESSION_BOOST_INFLIGHT_PER_POD` | `16` | Inflight requests admitted per backend pod; total inflight = perPod x backend pod count. Size it from the estimated per-pod concurrency (e.g. vLLM's --max-num-seqs) | -| `SESSION_BOOST_WAIT_REJECT_ENABLED` | `false` | Reject requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT` with HTTP 429. `FAIRNESS_QUEUE_TIMEOUT` does not apply in session-boost mode | +| Environment Variable | Default | Description | +| ----------------------------------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ENABLE_SESSION_BOOST` | `false` | Enable the session-boost scheduling strategy (mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING`) | +| `SESSION_BOOST_HEADER` | `X-Session-ID` | HTTP header used to identify conversation sessions | +| `SESSION_BOOST_MAX_SESSIONS` | `4096` | Maximum number of recently-completed sessions kept warm for boosting. Bounds an LRU cache; the least-recently-used session is evicted when exceeded. Sized by session count, not time | +| `SESSION_BOOST_GRACE_PERIOD` | `0` | Wait time after release for same-session follow-up. Disabled by default; enable only when you understand the latency trade-off | +| `SESSION_BOOST_INFLIGHT_PER_POD` | `16` | Inflight requests admitted per backend pod; total inflight = perPod x backend pod count. Size it from the estimated per-pod concurrency (e.g. vLLM's --max-num-seqs) | +| `SESSION_BOOST_WAIT_REJECT_ENABLED` | `false` | Reject requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT` with HTTP 429. `FAIRNESS_QUEUE_TIMEOUT` does not apply in session-boost mode | | `SESSION_BOOST_MAX_WAIT` | `30s` | Maximum time a request may wait in the queue before it is rejected with HTTP 429. Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`; the only server-side queue-wait bound in session-boost mode | ### Design Details From a585a2bde036d4cee6841907e96a3f5bc1061cf8 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Wed, 1 Jul 2026 08:08:12 +0000 Subject: [PATCH 04/11] fix comments Signed-off-by: YaoZengzeng --- pkg/kthena-router/router/router.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/pkg/kthena-router/router/router.go b/pkg/kthena-router/router/router.go index 29a6d0d0a..f5efb52d9 100644 --- a/pkg/kthena-router/router/router.go +++ b/pkg/kthena-router/router/router.go @@ -1185,16 +1185,32 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ // so the queue drops it from the heap (via its cancellation check), release // any permit if one was concurrently granted, and reject the client with 429. cancel() - if queueReq.Release != nil { - queueReq.Release() + // The dequeue goroutine writes queueReq.Release before closing NotifyChan, so a + // closed NotifyChan is the only thing that establishes the happens-before needed + // to read Release without a data race. A non-blocking receive lets us observe an + // admission that raced with this timeout and release its permit; if NotifyChan is + // not closed, admission never happened and Release is still nil. + select { + case <-queueReq.NotifyChan: + if queueReq.Release != nil { + queueReq.Release() + } + default: } klog.Warningf("[SessionBoost] request rejected after exceeding max queue wait: reqID=%s sessionID=%s user=%s model=%s maxWait=%v", requestID, sessionID, userId, modelName, r.sessionBoostMaxWait) c.AbortWithStatusJSON(http.StatusTooManyRequests, "Request rejected: exceeded maximum session boost queue wait time") return fmt.Errorf("request rejected: exceeded session boost max queue wait") case <-reqCtx.Done(): - if queueReq.Release != nil { - queueReq.Release() + // Same happens-before requirement as the waitReject path: only a closed + // NotifyChan guarantees we observe the dequeue goroutine's write to Release, so + // gate the read on a non-blocking receive to avoid a data race. + select { + case <-queueReq.NotifyChan: + if queueReq.Release != nil { + queueReq.Release() + } + default: } if errors.Is(reqCtx.Err(), context.DeadlineExceeded) { klog.Errorf("[FairnessScheduling] request timed out in queue: reqID=%s sessionID=%s user=%s model=%s timeout=%v", From 4e1a2ee83c86a92b3b76ac498eb12fba23a7ccf0 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Wed, 1 Jul 2026 08:12:42 +0000 Subject: [PATCH 05/11] address review comments: gate maxWait warning, quiet 429 log, fix docs, isolate test Signed-off-by: YaoZengzeng --- charts/kthena/charts/networking/values.yaml | 6 +++--- charts/kthena/values.yaml | 5 ++--- .../kthena/docs/reference/helm-chart-values.md | 4 ++-- pkg/kthena-router/router/router.go | 18 ++++++++++++++---- pkg/kthena-router/router/router_test.go | 10 +++++----- 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/charts/kthena/charts/networking/values.yaml b/charts/kthena/charts/networking/values.yaml index eaa7b7406..6915392df 100644 --- a/charts/kthena/charts/networking/values.yaml +++ b/charts/kthena/charts/networking/values.yaml @@ -71,12 +71,12 @@ kthenaRouter: # the latency trade-off for non-boosted requests. gracePeriod: "0s" # waitRejectEnabled controls whether requests that wait in the session-boost - # queue longer than maxWait are rejected with HTTP 429 instead of continuing to - # wait for the general queue timeout. Disabled by default. + # queue longer than maxWait are rejected with HTTP 429 instead of waiting + # indefinitely for backend capacity. Disabled by default. waitRejectEnabled: false # maxWait is the maximum time a request may wait in the session-boost queue # before it is rejected with HTTP 429. Only effective when waitRejectEnabled is - # true. Keep it below the general queue timeout so 429 fires before a 504. + # true. maxWait: "30s" # accessLog configuration for request logging accessLog: diff --git a/charts/kthena/values.yaml b/charts/kthena/values.yaml index 9fa824200..802c7e3c5 100644 --- a/charts/kthena/values.yaml +++ b/charts/kthena/values.yaml @@ -101,12 +101,11 @@ networking: # Disabled by default (`0s`). gracePeriod: "0s" # -- Reject requests that wait longer than `maxWait` in the session-boost
- # queue with HTTP 429 instead of waiting for the general queue timeout.
+ # queue with HTTP 429 instead of waiting indefinitely for backend capacity.
# Disabled by default. waitRejectEnabled: false # -- Maximum time a request may wait in the session-boost queue before it is
- # rejected with HTTP 429. Only effective when `waitRejectEnabled` is `true`.
- # Keep it below the general queue timeout so 429 fires before a 504. + # rejected with HTTP 429. Only effective when `waitRejectEnabled` is `true`. maxWait: "30s" gatewayAPI: # -- Enable Gateway API related features. diff --git a/docs/kthena/docs/reference/helm-chart-values.md b/docs/kthena/docs/reference/helm-chart-values.md index 9d567b0b7..124c46034 100644 --- a/docs/kthena/docs/reference/helm-chart-values.md +++ b/docs/kthena/docs/reference/helm-chart-values.md @@ -37,8 +37,8 @@ A Helm chart for deploying Kthena | networking.kthenaRouter.sessionBoost.header | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions. | | networking.kthenaRouter.sessionBoost.inflightPerPod | int | `16` | Maximum inflight requests admitted per backend pod.
Total inflight limit is this value times the number of backend pods.
`0` or unset uses the router default (16). | | networking.kthenaRouter.sessionBoost.maxSessions | int | `4096` | Maximum number of recently-completed sessions kept warm for boosting.
Bounds an LRU cache; the least-recently-used session is evicted automatically.
Size it by the number of concurrent conversations to keep boosted. | -| networking.kthenaRouter.sessionBoost.maxWait | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 429. Only effective when `waitRejectEnabled` is `true`.
Keep it below the general queue timeout so 429 fires before a 504. | -| networking.kthenaRouter.sessionBoost.waitRejectEnabled | bool | `false` | Reject requests that wait longer than `maxWait` in the session-boost
queue with HTTP 429 instead of waiting for the general queue timeout.
Disabled by default. | +| networking.kthenaRouter.sessionBoost.maxWait | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 429. Only effective when `waitRejectEnabled` is `true`. | +| networking.kthenaRouter.sessionBoost.waitRejectEnabled | bool | `false` | Reject requests that wait longer than `maxWait` in the session-boost
queue with HTTP 429 instead of waiting indefinitely for backend capacity.
Disabled by default. | | networking.kthenaRouter.terminationGracePeriodSeconds | int | `330` | The router will drain all in-flight requests before forcefully closing connections. | | networking.kthenaRouter.tls.dnsName | string | `"your-domain.com"` | DNS name to use for the certificate. | | networking.kthenaRouter.tls.enabled | bool | `false` | Enable TLS for Kthena Router server. | diff --git a/pkg/kthena-router/router/router.go b/pkg/kthena-router/router/router.go index f5efb52d9..90d03707d 100644 --- a/pkg/kthena-router/router/router.go +++ b/pkg/kthena-router/router/router.go @@ -214,8 +214,13 @@ const defaultSessionBoostMaxWait = 30 * time.Second // parseSessionBoostMaxWait reads the session-boost max queue wait from the // SESSION_BOOST_MAX_WAIT environment variable. It only takes effect when -// SESSION_BOOST_WAIT_REJECT_ENABLED is true. +// SESSION_BOOST_WAIT_REJECT_ENABLED is true, so when wait-reject is disabled we +// skip parsing entirely to avoid emitting confusing warnings for a value that +// has no effect. func parseSessionBoostMaxWait() time.Duration { + if !getEnvBool("SESSION_BOOST_WAIT_REJECT_ENABLED", false) { + return defaultSessionBoostMaxWait + } if s, ok := os.LookupEnv("SESSION_BOOST_MAX_WAIT"); ok { if d, err := time.ParseDuration(s); err == nil && d > 0 { return d @@ -1155,8 +1160,10 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ // Session-boost wait-reject: when enabled, a request that waits in the queue // longer than sessionBoostMaxWait is rejected with HTTP 429 instead of waiting - // out the general queue timeout (which returns 504). The timer only arms in - // session-boost mode when the feature is enabled and a positive max wait is set. + // indefinitely for backend capacity (session-boost mode has no + // FAIRNESS_QUEUE_TIMEOUT deadline, so there is no 504 fallback). The timer only + // arms in session-boost mode when the feature is enabled and a positive max wait + // is set. var waitRejectCh <-chan time.Time if EnableSessionBoost && r.sessionBoostWaitRejectEnabled && r.sessionBoostMaxWait > 0 { waitRejectTimer := time.NewTimer(r.sessionBoostMaxWait) @@ -1197,7 +1204,10 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ } default: } - klog.Warningf("[SessionBoost] request rejected after exceeding max queue wait: reqID=%s sessionID=%s user=%s model=%s maxWait=%v", + // 429 backpressure is expected behavior when wait-reject is enabled, and under + // sustained overload this path can fire for many requests, so log at a verbose + // level (like the rate-limit 429 path) to avoid flooding warning logs. + klog.V(2).Infof("[SessionBoost] request rejected after exceeding max queue wait: reqID=%s sessionID=%s user=%s model=%s maxWait=%v", requestID, sessionID, userId, modelName, r.sessionBoostMaxWait) c.AbortWithStatusJSON(http.StatusTooManyRequests, "Request rejected: exceeded maximum session boost queue wait time") return fmt.Errorf("request rejected: exceeded session boost max queue wait") diff --git a/pkg/kthena-router/router/router_test.go b/pkg/kthena-router/router/router_test.go index 9af0f4a4e..d3551c50b 100644 --- a/pkg/kthena-router/router/router_test.go +++ b/pkg/kthena-router/router/router_test.go @@ -1438,11 +1438,11 @@ func TestHandleFairnessScheduling(t *testing.T) { if tt.sessionBoostMaxWait > 0 { router.sessionBoostMaxWait = tt.sessionBoostMaxWait } - if tt.enableSessionBoost { - prev := EnableSessionBoost - EnableSessionBoost = true - defer func() { EnableSessionBoost = prev }() - } + // Set the package-level flag explicitly for every case (and restore it) + // so subtests stay isolated regardless of execution order. + prevEnableSessionBoost := EnableSessionBoost + EnableSessionBoost = tt.enableSessionBoost + defer func() { EnableSessionBoost = prevEnableSessionBoost }() if tt.storeWrapper != nil { router.store = tt.storeWrapper(store) } From 8b70dce1b99b29e0770b6d50c3188715eff18f8e Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Tue, 7 Jul 2026 03:50:05 +0000 Subject: [PATCH 06/11] fix comments Signed-off-by: YaoZengzeng --- pkg/kthena-router/datastore/fairness_queue.go | 83 ++++++++++-- .../datastore/session_boost_queue.go | 61 ++++++--- .../datastore/session_boost_queue_test.go | 121 ++++++++++++++++++ pkg/kthena-router/router/router.go | 38 +++--- 4 files changed, 246 insertions(+), 57 deletions(-) diff --git a/pkg/kthena-router/datastore/fairness_queue.go b/pkg/kthena-router/datastore/fairness_queue.go index 02c3f0c78..25a6a47cb 100644 --- a/pkg/kthena-router/datastore/fairness_queue.go +++ b/pkg/kthena-router/datastore/fairness_queue.go @@ -152,6 +152,51 @@ type Request struct { NotifyChan chan struct{} CancelCh <-chan struct{} // Request-scoped cancellation signal Release func() // Set by the queue when a permit is acquired + + // admitMu serializes admission (by the dequeue loop) against abandonment (by + // the waiting caller on timeout/cancel), closing the race where the loop has + // popped the request and passed its cancellation check but has not yet + // installed Release / closed NotifyChan. Guards admitted and abandoned. + admitMu sync.Mutex + admitted bool // admission committed: Release is set and about to be signalled + abandoned bool // caller gave up before admission; the loop must not admit +} + +// commitAdmission runs fn under the request lock, but only if the caller has not +// already abandoned the request. fn performs the admission side effects that must +// be fully visible before the request is considered admitted: acquiring the +// inflight permit, installing Release, and incrementing the inflight metric. +// It returns true if admission was committed. When it returns false the request +// was abandoned first, so the dequeue loop must not mark it inflight or signal it. +// +// Because fn (including installing Release) completes before admitted is set, any +// caller that observes admitted via abandon() is guaranteed to see a non-nil +// Release, so the permit can always be returned. +func (r *Request) commitAdmission(fn func()) bool { + r.admitMu.Lock() + defer r.admitMu.Unlock() + if r.abandoned { + return false + } + fn() + r.admitted = true + return true +} + +// Abandon marks the request as given up by the waiting caller (queue timeout, +// wait-reject, or client cancellation). It returns true if the request had +// already been admitted, in which case the caller owns the inflight permit and +// MUST call Release to avoid leaking capacity. When it returns false, admission is +// guaranteed not to proceed (commitAdmission observes abandoned and skips), so no +// permit can leak. +func (r *Request) Abandon() bool { + r.admitMu.Lock() + defer r.admitMu.Unlock() + if r.admitted { + return true + } + r.abandoned = true + return false } // RequestPriorityQueue implements the heap.Interface @@ -485,21 +530,31 @@ func (pq *RequestPriorityQueue) runSemaphoreMode(ctx context.Context) { // Permit acquired } - releaseOnce := sync.Once{} - trackedInflight := false - req.Release = func() { - releaseOnce.Do(func() { - <-pq.sem - if trackedInflight && pq.metrics != nil { - pq.metrics.DecFairnessQueueInflight(req.ModelName) - } - }) - } - - if pq.metrics != nil { - pq.metrics.IncFairnessQueueInflight(req.ModelName) + // Commit admission atomically with respect to a concurrent Abandon(): install + // Release and increment the inflight metric under the request lock so the + // waiting caller either observes the admission (and releases the permit on + // timeout) or blocks admission entirely. Increment the metric inside the + // committed section so a racing Release always has a matching increment. + admitted := req.commitAdmission(func() { + releaseOnce := sync.Once{} + req.Release = func() { + releaseOnce.Do(func() { + <-pq.sem + if pq.metrics != nil { + pq.metrics.DecFairnessQueueInflight(req.ModelName) + } + }) + } + if pq.metrics != nil { + pq.metrics.IncFairnessQueueInflight(req.ModelName) + } + }) + if !admitted { + // Caller abandoned before admission; return the just-acquired permit and + // drop the request without signalling it. + <-pq.sem + continue } - trackedInflight = true close(req.NotifyChan) } } diff --git a/pkg/kthena-router/datastore/session_boost_queue.go b/pkg/kthena-router/datastore/session_boost_queue.go index e6aa13bc6..95ee4d5dd 100644 --- a/pkg/kthena-router/datastore/session_boost_queue.go +++ b/pkg/kthena-router/datastore/session_boost_queue.go @@ -124,27 +124,40 @@ func (pq *RequestPriorityQueue) GetInflightCount() int64 { } // admitSessionBoost marks a request as inflight, installs its release callback and -// unblocks the waiting caller by closing its NotifyChan. -func (pq *RequestPriorityQueue) admitSessionBoost(req *Request) { - pq.inflightCount.Add(1) - // req.Release returns the inflight permit this admission consumed. The request - // handler invokes it (via defer) once the backend response is fully proxied or - // the request fails/times out. It decrements the inflight count, signals - // releaseCh so the dequeue loop can immediately admit the next waiting request, - // and updates the inflight metric. sync.Once makes it idempotent so capacity is - // never released twice on overlapping exit paths. - releaseOnce := sync.Once{} - req.Release = func() { - releaseOnce.Do(func() { - pq.inflightCount.Add(-1) - select { - case pq.releaseCh <- struct{}{}: - default: - } - pq.metricDecInflight(req.ModelName) - }) +// unblocks the waiting caller by closing its NotifyChan. It returns false without +// admitting when the caller has already abandoned the request (timed out or +// cancelled): in that case the request has already left the queue and admitting it +// would leak an inflight permit that no one will release. +func (pq *RequestPriorityQueue) admitSessionBoost(req *Request) bool { + // Commit the inflight accounting and Release installation atomically with + // respect to a concurrent Abandon(). If the caller abandoned first, skip + // admission entirely so the inflight permit is never consumed. + admitted := req.commitAdmission(func() { + pq.inflightCount.Add(1) + // req.Release returns the inflight permit this admission consumed. The request + // handler invokes it (via defer) once the backend response is fully proxied or + // the request fails/times out. It decrements the inflight count, signals + // releaseCh so the dequeue loop can immediately admit the next waiting request, + // and updates the inflight metric. sync.Once makes it idempotent so capacity is + // never released twice on overlapping exit paths. + releaseOnce := sync.Once{} + req.Release = func() { + releaseOnce.Do(func() { + pq.inflightCount.Add(-1) + select { + case pq.releaseCh <- struct{}{}: + default: + } + pq.metricDecInflight(req.ModelName) + }) + } + pq.metricIncInflight(req.ModelName) + }) + if !admitted { + klog.V(4).Infof("[SessionBoost] admission skipped, request abandoned before admission: user=%s model=%s", + req.UserID, req.ModelName) + return false } - pq.metricIncInflight(req.ModelName) // Closing NotifyChan is the admission signal: the caller blocked in Enqueue is // waiting on this channel and proceeds to the backend only once it is closed. // We notify here because admission (a free inflight slot plus backend capacity) @@ -153,6 +166,7 @@ func (pq *RequestPriorityQueue) admitSessionBoost(req *Request) { if req.NotifyChan != nil { close(req.NotifyChan) } + return true } // runSessionBoostMode is the session-boost dequeue loop. It dequeues requests only @@ -347,7 +361,12 @@ func (pq *RequestPriorityQueue) tryBackpressureDequeue(ctx context.Context) { return } - pq.admitSessionBoost(req) + if !pq.admitSessionBoost(req) { + // The request was abandoned (timed out / cancelled) between the + // cancellation check and admission; it consumed no inflight slot, so + // continue admitting the next queued request. + continue + } klog.V(4).Infof("[SessionBoost] backpressure dequeue: user=%s model=%s sessionBoost=%v inflight=%d/%d", req.UserID, req.ModelName, req.SessionBoost, pq.inflightCount.Load(), maxInflight) diff --git a/pkg/kthena-router/datastore/session_boost_queue_test.go b/pkg/kthena-router/datastore/session_boost_queue_test.go index 36faacb3c..bd52f75d4 100644 --- a/pkg/kthena-router/datastore/session_boost_queue_test.go +++ b/pkg/kthena-router/datastore/session_boost_queue_test.go @@ -18,6 +18,7 @@ package datastore import ( "context" + "sync" "testing" "time" ) @@ -556,3 +557,123 @@ func TestSessionBoostQueue_Len(t *testing.T) { t.Errorf("Expected len=5, got %d", q.Len()) } } + +// TestSessionBoost_AbandonBeforeAdmissionBlocksAdmission verifies that a request +// abandoned by the caller before the dequeue loop admits it consumes no inflight +// slot: admitSessionBoost must observe the abandonment and skip. +func TestSessionBoost_AbandonBeforeAdmissionBlocksAdmission(t *testing.T) { + cfg := sessionBoostConfig() + q := newSessionBoostQueue(cfg, nil) + defer q.Close() + + req := &Request{ + UserID: "user-A", + ModelName: "model-1", + NotifyChan: make(chan struct{}), + } + + // Caller gives up first. + if admitted := req.Abandon(); admitted { + t.Fatal("Abandon() should report not-yet-admitted before admission") + } + + // The dequeue loop then tries to admit; it must skip. + if q.admitSessionBoost(req) { + t.Fatal("admitSessionBoost should skip an already-abandoned request") + } + if got := q.GetInflightCount(); got != 0 { + t.Fatalf("inflight count should stay 0 after skipped admission, got %d", got) + } + select { + case <-req.NotifyChan: + t.Fatal("NotifyChan must not be closed for a skipped admission") + default: + } +} + +// TestSessionBoost_AdmissionBeforeAbandonReleases verifies that when admission wins +// the race, the caller's Abandon() reports the admission and can release the permit, +// returning the inflight count to zero. +func TestSessionBoost_AdmissionBeforeAbandonReleases(t *testing.T) { + cfg := sessionBoostConfig() + q := newSessionBoostQueue(cfg, nil) + defer q.Close() + + req := &Request{ + UserID: "user-A", + ModelName: "model-1", + NotifyChan: make(chan struct{}), + } + + if !q.admitSessionBoost(req) { + t.Fatal("admitSessionBoost should admit a live request") + } + select { + case <-req.NotifyChan: + default: + t.Fatal("NotifyChan should be closed after admission") + } + if got := q.GetInflightCount(); got != 1 { + t.Fatalf("inflight count should be 1 after admission, got %d", got) + } + + // Caller times out after admission raced in: Abandon() must report admitted so + // the caller releases the permit. + if admitted := req.Abandon(); !admitted { + t.Fatal("Abandon() should report admitted after admission") + } + if req.Release == nil { + t.Fatal("Release must be installed once Abandon() reports admitted") + } + req.Release() + if got := q.GetInflightCount(); got != 0 { + t.Fatalf("inflight count should return to 0 after Release, got %d", got) + } +} + +// TestSessionBoost_ConcurrentAdmitAbandonNoLeak stresses the admission/abandonment +// race directly: for each request one goroutine admits while another abandons. No +// matter which wins, the inflight count must return to zero, proving the permit is +// never leaked and never double-released. Run with -race to catch data races. +func TestSessionBoost_ConcurrentAdmitAbandonNoLeak(t *testing.T) { + cfg := sessionBoostConfig() + q := newSessionBoostQueue(cfg, nil) + defer q.Close() + + const iterations = 2000 + for i := 0; i < iterations; i++ { + req := &Request{ + UserID: "user-A", + ModelName: "model-1", + NotifyChan: make(chan struct{}), + } + + var wg sync.WaitGroup + wg.Add(2) + // Dequeue-loop side: attempt admission. + go func() { + defer wg.Done() + q.admitSessionBoost(req) + }() + // Handler side: time out and abandon; release if admission raced in. This + // mirrors the router's waitReject/timeout handling. + go func() { + defer wg.Done() + if req.Abandon() { + req.Release() + } + }() + wg.Wait() + + // After both settle, drain any releaseCh signal the release may have posted so + // it does not affect the next iteration's assertions. + select { + case <-q.releaseCh: + default: + } + + if got := q.GetInflightCount(); got != 0 { + t.Fatalf("iteration %d: inflight leaked, count=%d", i, got) + } + } +} diff --git a/pkg/kthena-router/router/router.go b/pkg/kthena-router/router/router.go index 90d03707d..e693aa782 100644 --- a/pkg/kthena-router/router/router.go +++ b/pkg/kthena-router/router/router.go @@ -1189,20 +1189,17 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ return nil case <-waitRejectCh: // Exceeded the session-boost maximum queue wait. Cancel the request context - // so the queue drops it from the heap (via its cancellation check), release - // any permit if one was concurrently granted, and reject the client with 429. + // so the queue drops it from the heap (via its cancellation check), then + // abandon it to reject the client with 429. cancel() - // The dequeue goroutine writes queueReq.Release before closing NotifyChan, so a - // closed NotifyChan is the only thing that establishes the happens-before needed - // to read Release without a data race. A non-blocking receive lets us observe an - // admission that raced with this timeout and release its permit; if NotifyChan is - // not closed, admission never happened and Release is still nil. - select { - case <-queueReq.NotifyChan: - if queueReq.Release != nil { - queueReq.Release() - } - default: + // Abandon() atomically coordinates with the dequeue loop: if admission raced + // in first it returns true and we own the inflight permit, so release it here; + // otherwise it marks the request abandoned so the loop skips admission and no + // permit can leak. This replaces the previous non-blocking NotifyChan receive, + // which could miss an admission that had passed its cancellation check but not + // yet closed NotifyChan, leaking the permit. + if queueReq.Abandon() { + queueReq.Release() } // 429 backpressure is expected behavior when wait-reject is enabled, and under // sustained overload this path can fire for many requests, so log at a verbose @@ -1212,15 +1209,12 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ c.AbortWithStatusJSON(http.StatusTooManyRequests, "Request rejected: exceeded maximum session boost queue wait time") return fmt.Errorf("request rejected: exceeded session boost max queue wait") case <-reqCtx.Done(): - // Same happens-before requirement as the waitReject path: only a closed - // NotifyChan guarantees we observe the dequeue goroutine's write to Release, so - // gate the read on a non-blocking receive to avoid a data race. - select { - case <-queueReq.NotifyChan: - if queueReq.Release != nil { - queueReq.Release() - } - default: + // Same admission/abandonment coordination as the waitReject path: Abandon() + // either observes a concurrent admission (returning true so we release the + // permit) or blocks admission, so a timeout/cancel can never leak an inflight + // permit. + if queueReq.Abandon() { + queueReq.Release() } if errors.Is(reqCtx.Err(), context.DeadlineExceeded) { klog.Errorf("[FairnessScheduling] request timed out in queue: reqID=%s sessionID=%s user=%s model=%s timeout=%v", From 473200d16dec0f657d0b12d4395a5f0c83f8ca4a Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Tue, 7 Jul 2026 07:48:13 +0000 Subject: [PATCH 07/11] fix comments Signed-off-by: YaoZengzeng --- charts/kthena/charts/networking/README.md | 4 ++-- charts/kthena/charts/networking/values.yaml | 4 ++-- charts/kthena/values.yaml | 4 ++-- docs/kthena/docs/user-guide/session-boost.md | 24 +++++++++---------- docs/proposal/session-boost-strategy.md | 14 +++++------ pkg/kthena-router/router/router.go | 25 ++++++++++---------- pkg/kthena-router/router/router_test.go | 4 ++-- 7 files changed, 40 insertions(+), 39 deletions(-) diff --git a/charts/kthena/charts/networking/README.md b/charts/kthena/charts/networking/README.md index 88307c6c0..aabc45c88 100644 --- a/charts/kthena/charts/networking/README.md +++ b/charts/kthena/charts/networking/README.md @@ -64,8 +64,8 @@ kthenaRouter: | `kthenaRouter.sessionBoost.maxSessions` | int | `4096` | Max recently-completed sessions kept warm (LRU-evicted) | | `kthenaRouter.sessionBoost.inflightPerPod` | int | `16` | Inflight requests per backend pod; total = perPod x pod count | | `kthenaRouter.sessionBoost.gracePeriod` | string | `"0s"` | Wait time for a same-session follow-up (disabled by default) | -| `kthenaRouter.sessionBoost.waitRejectEnabled` | boolean | `false` | Reject requests waiting longer than `maxWait` with HTTP 429 | -| `kthenaRouter.sessionBoost.maxWait` | string | `"30s"` | Max queue wait before 429 (only when `waitRejectEnabled: true`) | +| `kthenaRouter.sessionBoost.waitRejectEnabled` | boolean | `false` | Reject requests waiting longer than `maxWait` with HTTP 504 | +| `kthenaRouter.sessionBoost.maxWait` | string | `"30s"` | Max queue wait before 504 (only when `waitRejectEnabled: true`) | #### Session Boost Configuration diff --git a/charts/kthena/charts/networking/values.yaml b/charts/kthena/charts/networking/values.yaml index 6915392df..4318820c0 100644 --- a/charts/kthena/charts/networking/values.yaml +++ b/charts/kthena/charts/networking/values.yaml @@ -71,11 +71,11 @@ kthenaRouter: # the latency trade-off for non-boosted requests. gracePeriod: "0s" # waitRejectEnabled controls whether requests that wait in the session-boost - # queue longer than maxWait are rejected with HTTP 429 instead of waiting + # queue longer than maxWait are rejected with HTTP 504 instead of waiting # indefinitely for backend capacity. Disabled by default. waitRejectEnabled: false # maxWait is the maximum time a request may wait in the session-boost queue - # before it is rejected with HTTP 429. Only effective when waitRejectEnabled is + # before it is rejected with HTTP 504. Only effective when waitRejectEnabled is # true. maxWait: "30s" # accessLog configuration for request logging diff --git a/charts/kthena/values.yaml b/charts/kthena/values.yaml index 802c7e3c5..9aed836f8 100644 --- a/charts/kthena/values.yaml +++ b/charts/kthena/values.yaml @@ -101,11 +101,11 @@ networking: # Disabled by default (`0s`). gracePeriod: "0s" # -- Reject requests that wait longer than `maxWait` in the session-boost
- # queue with HTTP 429 instead of waiting indefinitely for backend capacity.
+ # queue with HTTP 504 instead of waiting indefinitely for backend capacity.
# Disabled by default. waitRejectEnabled: false # -- Maximum time a request may wait in the session-boost queue before it is
- # rejected with HTTP 429. Only effective when `waitRejectEnabled` is `true`. + # rejected with HTTP 504. Only effective when `waitRejectEnabled` is `true`. maxWait: "30s" gatewayAPI: # -- Enable Gateway API related features. diff --git a/docs/kthena/docs/user-guide/session-boost.md b/docs/kthena/docs/user-guide/session-boost.md index e85c56cc0..890dfdd37 100644 --- a/docs/kthena/docs/user-guide/session-boost.md +++ b/docs/kthena/docs/user-guide/session-boost.md @@ -31,7 +31,7 @@ flowchart TD F -- Yes --> G[Dispatch to backend] F -- No --> H[Wait / backpressure] H --> K{Waited longer than
SESSION_BOOST_MAX_WAIT?} - K -- Yes, wait-reject enabled --> R[Reject with HTTP 429] + K -- Yes, wait-reject enabled --> R[Reject with HTTP 504] K -- No --> F G --> I[Request completes] I --> J[Mark session completed
promote in LRU cache] @@ -75,8 +75,8 @@ networking: header: "X-Session-ID" maxSessions: 4096 # LRU cache of recently-completed sessions kept warm inflightPerPod: 16 # total inflight = inflightPerPod x backend pod count - waitRejectEnabled: true # reject requests that wait too long with HTTP 429 - maxWait: "30s" # max queue wait before a 429 is returned + waitRejectEnabled: true # reject requests that wait too long with HTTP 504 + maxWait: "30s" # max queue wait before a 504 is returned ``` Apply with Helm: @@ -103,9 +103,9 @@ env: - name: SESSION_BOOST_INFLIGHT_PER_POD value: "16" # total inflight = perPod x backend pod count - name: SESSION_BOOST_WAIT_REJECT_ENABLED - value: "true" # reject requests that wait too long with HTTP 429 + value: "true" # reject requests that wait too long with HTTP 504 - name: SESSION_BOOST_MAX_WAIT - value: "30s" # max queue wait before a 429 is returned + value: "30s" # max queue wait before a 504 is returned ``` ## Configuration Reference @@ -116,8 +116,8 @@ env: | `SESSION_BOOST_HEADER` | HTTP header used to identify conversation sessions | `X-Session-ID` | Must match what your clients send | | `SESSION_BOOST_MAX_SESSIONS` | Max number of recently-completed sessions kept "warm" for boosting (LRU-bounded) | `4096` | When the cache is full, the least-recently-used session is evicted automatically. Size it by the number of concurrent conversations you want to keep boosted—no time-based tuning required | | `SESSION_BOOST_INFLIGHT_PER_POD` | Inflight requests admitted per backend pod | `16` | The total inflight limit is this value multiplied by the number of backend pods. Size it from the per-pod concurrency (e.g., vLLM's `--max-num-seqs`) | -| `SESSION_BOOST_WAIT_REJECT_ENABLED` | Reject requests that wait too long in the queue with HTTP 429 | `false` | When `true`, a request queued longer than `SESSION_BOOST_MAX_WAIT` is rejected with `429 Too Many Requests` instead of continuing to wait | -| `SESSION_BOOST_MAX_WAIT` | Maximum time a request may wait in the queue before it is rejected with 429 | `30s` | Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. This is the only server-side queue-wait bound in session-boost mode; `FAIRNESS_QUEUE_TIMEOUT` does not apply | +| `SESSION_BOOST_WAIT_REJECT_ENABLED` | Reject requests that wait too long in the queue with HTTP 504 | `false` | When `true`, a request queued longer than `SESSION_BOOST_MAX_WAIT` is rejected with `504 Gateway Timeout` instead of continuing to wait | +| `SESSION_BOOST_MAX_WAIT` | Maximum time a request may wait in the queue before it is rejected with 504 | `30s` | Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. This is the only server-side queue-wait bound in session-boost mode; `FAIRNESS_QUEUE_TIMEOUT` does not apply | > `SESSION_BOOST_GRACE_PERIOD` is intentionally omitted from the table above. It is an advanced, scenario-specific knob that is disabled by default; see [Advanced: Grace Period](#advanced-grace-period-use-with-caution). @@ -184,16 +184,16 @@ The queue uses two-level admission control to avoid flooding backends: When a request completes, the queue immediately attempts to dequeue the next request (release-driven dequeue) rather than waiting for the next metrics refresh. -### Queue Wait Timeout (429 Rejection) +### Queue Wait Timeout (504 Rejection) Under heavy load a request may sit in the queue for a long time before backend capacity frees up. Session boost does **not** apply the fairness queue timeout (`FAIRNESS_QUEUE_TIMEOUT` only governs the user-fairness queue); by default a queued request in session-boost mode waits until backend capacity frees up or the client disconnects. For latency-sensitive front ends it is often better to **fail fast** and let the client retry or shed load. -Session boost provides its own optional **wait timeout** that rejects over-queued requests early with `429 Too Many Requests`: +Session boost provides its own optional **wait timeout** that rejects over-queued requests early with `504 Gateway Timeout`: - Enable it with `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. - Set the threshold with `SESSION_BOOST_MAX_WAIT` (default `30s`). -When enabled, if a request waits in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router removes it from the queue and responds with `429 Too Many Requests`. The 429 signals *backpressure* (the queue is saturated and the client should back off or retry). +When enabled, if a request waits in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router removes it from the queue and responds with `504 Gateway Timeout`. The 504 signals that the request *timed out* waiting for backend capacity (the queue is saturated); the client can retry later or shed load. > `SESSION_BOOST_MAX_WAIT` is the only server-side queue-wait bound in session-boost mode. `FAIRNESS_QUEUE_TIMEOUT` has no effect here—it applies exclusively to the user-fairness strategy. Without `SESSION_BOOST_WAIT_REJECT_ENABLED`, a session-boost request is bounded only by client disconnect. @@ -272,9 +272,9 @@ Session boost only controls queue ordering. If the boosted request is routed to Each tracked session consumes minimal memory (just a session ID). The tracker is a bounded LRU cache, so total memory is capped at `SESSION_BOOST_MAX_SESSIONS` entries regardless of how much traffic flows through the router. If memory is a concern, reduce `SESSION_BOOST_MAX_SESSIONS`. -### Clients receive HTTP 429 responses +### Clients receive HTTP 504 responses -When `SESSION_BOOST_WAIT_REJECT_ENABLED=true`, the router returns `429 Too Many Requests` for requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT`. This is expected backpressure behavior under sustained overload—it means the backends cannot keep up with demand. If you see more 429s than desired: +When `SESSION_BOOST_WAIT_REJECT_ENABLED=true`, the router returns `504 Gateway Timeout` for requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT`. This is expected load-shedding under sustained overload—it means the backends cannot keep up with demand. If you see more 504s than desired: 1. Confirm the backends are healthy and scaled appropriately; add capacity if they are saturated. 2. Increase `SESSION_BOOST_MAX_WAIT` to allow requests to wait longer before rejection. diff --git a/docs/proposal/session-boost-strategy.md b/docs/proposal/session-boost-strategy.md index 1a17103a2..8126f7377 100644 --- a/docs/proposal/session-boost-strategy.md +++ b/docs/proposal/session-boost-strategy.md @@ -39,7 +39,7 @@ Enabling session boost therefore reconfigures the same per-model priority queue 3. **KV cache optimization**: Prioritize follow-up requests from recently completed sessions to maximize warm cache hits. 4. **Grace period (advanced, off by default)**: An optional, tricky tuning knob that, after a request completes, briefly holds the dequeue slot for a potential follow-up from the same session before dispatching unrelated requests. It is **disabled by default** and should only be enabled by operators who fully understand that it deliberately delays unrelated requests in exchange for a higher same-session prefix-cache hit rate. 5. **Backpressure-aware**: Respect backend pod capacity to avoid flooding, using two-level admission control (inflight limit + backend metrics). -6. **Fail-fast queue wait timeout (optional, off by default)**: Optionally reject requests that wait in the queue longer than a configurable threshold with HTTP 429, so latency-sensitive clients shed load or retry instead of waiting indefinitely for backend capacity. `FAIRNESS_QUEUE_TIMEOUT` governs only the user-fairness queue and does not apply in session-boost mode. +6. **Fail-fast queue wait timeout (optional, off by default)**: Optionally reject requests that wait in the queue longer than a configurable threshold with HTTP 504, so latency-sensitive clients shed load or retry instead of waiting indefinitely for backend capacity. `FAIRNESS_QUEUE_TIMEOUT` governs only the user-fairness queue and does not apply in session-boost mode. #### Non-Goals @@ -226,8 +226,8 @@ The net effect is a strict precedence: **grace timing → inflight gate → back | `SESSION_BOOST_MAX_SESSIONS` | `4096` | Maximum number of recently-completed sessions kept warm for boosting. Bounds an LRU cache; the least-recently-used session is evicted when exceeded. Sized by session count, not time | | `SESSION_BOOST_GRACE_PERIOD` | `0` | Wait time after release for same-session follow-up. Disabled by default; enable only when you understand the latency trade-off | | `SESSION_BOOST_INFLIGHT_PER_POD` | `16` | Inflight requests admitted per backend pod; total inflight = perPod x backend pod count. Size it from the estimated per-pod concurrency (e.g. vLLM's --max-num-seqs) | -| `SESSION_BOOST_WAIT_REJECT_ENABLED` | `false` | Reject requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT` with HTTP 429. `FAIRNESS_QUEUE_TIMEOUT` does not apply in session-boost mode | -| `SESSION_BOOST_MAX_WAIT` | `30s` | Maximum time a request may wait in the queue before it is rejected with HTTP 429. Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`; the only server-side queue-wait bound in session-boost mode | +| `SESSION_BOOST_WAIT_REJECT_ENABLED` | `false` | Reject requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT` with HTTP 504. `FAIRNESS_QUEUE_TIMEOUT` does not apply in session-boost mode | +| `SESSION_BOOST_MAX_WAIT` | `30s` | Maximum time a request may wait in the queue before it is rejected with HTTP 504. Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`; the only server-side queue-wait bound in session-boost mode | ### Design Details @@ -300,7 +300,7 @@ The queue uses two-level admission control: When a request completes (Release), the queue immediately attempts to dequeue the next request (release-driven dequeue), ensuring minimal latency between sequential requests. The loop is fully event-driven — there is no independent polling timer. In single-router operation every moment a backend frees capacity coincides with one of our own requests completing (a release), so release and arrival events alone cover every dequeue opportunity; the capacity check simply reads the pod metrics already scraped by the store (`METRICS_SCRAPE_INTERVAL`). -#### Queue Wait Timeout (429 Rejection) +#### Queue Wait Timeout (504 Rejection) Under sustained overload the two-level admission control legitimately holds requests in the queue until backend capacity frees up. `FAIRNESS_QUEUE_TIMEOUT` governs **only** the user-fairness queue and does not apply in session-boost mode, so by default a queued session-boost request waits until backend capacity frees up or the client disconnects (which returns `503`). For latency-sensitive front ends this open-ended wait can be undesirable: the client would rather be told *quickly* that the system is saturated so it can retry elsewhere or shed load. @@ -308,12 +308,12 @@ The optional queue wait timeout provides this fail-fast behavior: - It is enabled with `SESSION_BOOST_WAIT_REJECT_ENABLED=true` and disabled by default. - The threshold is configured with `SESSION_BOOST_MAX_WAIT` (default `30s`). -- When a request has been waiting in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router stops waiting, removes the request from the queue, and responds with `429 Too Many Requests`. +- When a request has been waiting in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router stops waiting, removes the request from the queue, and responds with `504 Gateway Timeout`. -It is implemented in the router's request-handling path rather than in the queue's dequeue loop, mirroring how the fairness queue's own `FAIRNESS_QUEUE_TIMEOUT` (504) is handled. In session-boost mode the request context is created **without** the fairness deadline, so `FAIRNESS_QUEUE_TIMEOUT` has no effect. When the feature is on, the handler arms a timer for `SESSION_BOOST_MAX_WAIT` alongside the admission (`NotifyChan`) and cancellation (`reqCtx.Done()`) signals it already waits on: +It is implemented in the router's request-handling path rather than in the queue's dequeue loop, mirroring how the fairness queue's own `FAIRNESS_QUEUE_TIMEOUT` (504) is handled. Both queue-wait timeouts therefore return the same `504` status; they differ only in what arms them (`FAIRNESS_QUEUE_TIMEOUT` vs `SESSION_BOOST_MAX_WAIT`). In session-boost mode the request context is created **without** the fairness deadline, so `FAIRNESS_QUEUE_TIMEOUT` has no effect. When the feature is on, the handler arms a timer for `SESSION_BOOST_MAX_WAIT` alongside the admission (`NotifyChan`) and cancellation (`reqCtx.Done()`) signals it already waits on: - **Admitted first**: the request proceeds to the backend as usual (no rejection). -- **Wait timer fires first**: the handler cancels the request context — which makes the queue drop the request from the heap via its existing cancellation check (`isCancelled` / `drainCancelledLocked`), reusing the same cleanup path as client disconnects — releases any permit that may have been granted concurrently, and returns `429 Too Many Requests`. +- **Wait timer fires first**: the handler cancels the request context — which makes the queue drop the request from the heap via its existing cancellation check (`isCancelled` / `drainCancelledLocked`), reusing the same cleanup path as client disconnects — releases any permit that may have been granted concurrently, and returns `504 Gateway Timeout`. - **Client disconnect fires first**: the existing behavior is unchanged (`503`). Because it reuses the queue's cancellation cleanup, the wait timeout adds no new state to the queue and no extra polling. It is orthogonal to the inflight and backend capacity gates: those gates decide *whether* a request may run, while the wait timeout only bounds *how long* a request is willing to wait before giving up. `SESSION_BOOST_MAX_WAIT` is the only server-side queue-wait bound in session-boost mode; the feature only applies in session-boost mode. diff --git a/pkg/kthena-router/router/router.go b/pkg/kthena-router/router/router.go index e693aa782..90af2978e 100644 --- a/pkg/kthena-router/router/router.go +++ b/pkg/kthena-router/router/router.go @@ -95,7 +95,7 @@ type Router struct { // Session-boost wait-reject configuration. When sessionBoostWaitRejectEnabled // is true, a request that waits in the session-boost queue longer than - // sessionBoostMaxWait is rejected with HTTP 429 instead of continuing to wait + // sessionBoostMaxWait is rejected with HTTP 504 instead of continuing to wait // for the general queue timeout. sessionBoostWaitRejectEnabled bool sessionBoostMaxWait time.Duration @@ -208,7 +208,7 @@ func parseQueueTimeout() time.Duration { } // defaultSessionBoostMaxWait is the maximum time a request may wait in the -// session-boost queue before it is rejected with HTTP 429, used when +// session-boost queue before it is rejected with HTTP 504, used when // SESSION_BOOST_MAX_WAIT is not set or invalid and wait-reject is enabled. const defaultSessionBoostMaxWait = 30 * time.Second @@ -1117,7 +1117,7 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ // cleanup (CancelCh). The general queue-wait deadline differs by strategy: // - Fairness mode: bounded by FAIRNESS_QUEUE_TIMEOUT; exceeding it returns 504. // - Session-boost mode: FAIRNESS_QUEUE_TIMEOUT does NOT apply. Session boost has - // its own independent wait control via SESSION_BOOST_MAX_WAIT (returns 429 when + // its own independent wait control via SESSION_BOOST_MAX_WAIT (returns 504 when // SESSION_BOOST_WAIT_REJECT_ENABLED is set); otherwise the request is bounded // only by client disconnect. var reqCtx context.Context @@ -1159,11 +1159,11 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ } // Session-boost wait-reject: when enabled, a request that waits in the queue - // longer than sessionBoostMaxWait is rejected with HTTP 429 instead of waiting + // longer than sessionBoostMaxWait is rejected with HTTP 504 instead of waiting // indefinitely for backend capacity (session-boost mode has no - // FAIRNESS_QUEUE_TIMEOUT deadline, so there is no 504 fallback). The timer only - // arms in session-boost mode when the feature is enabled and a positive max wait - // is set. + // FAIRNESS_QUEUE_TIMEOUT deadline, so this wait-reject timer is the only + // queue-wait bound). The timer only arms in session-boost mode when the feature + // is enabled and a positive max wait is set. var waitRejectCh <-chan time.Time if EnableSessionBoost && r.sessionBoostWaitRejectEnabled && r.sessionBoostMaxWait > 0 { waitRejectTimer := time.NewTimer(r.sessionBoostMaxWait) @@ -1190,7 +1190,7 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ case <-waitRejectCh: // Exceeded the session-boost maximum queue wait. Cancel the request context // so the queue drops it from the heap (via its cancellation check), then - // abandon it to reject the client with 429. + // abandon it to reject the client with 504. cancel() // Abandon() atomically coordinates with the dequeue loop: if admission raced // in first it returns true and we own the inflight permit, so release it here; @@ -1201,12 +1201,13 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ if queueReq.Abandon() { queueReq.Release() } - // 429 backpressure is expected behavior when wait-reject is enabled, and under - // sustained overload this path can fire for many requests, so log at a verbose - // level (like the rate-limit 429 path) to avoid flooding warning logs. + // A session-boost wait timeout is expected load-shedding when wait-reject is + // enabled, and under sustained overload this path can fire for many requests, so + // log at a verbose level to avoid flooding the logs (the fairness 504 timeout, + // by contrast, is unexpected and logs at error level). klog.V(2).Infof("[SessionBoost] request rejected after exceeding max queue wait: reqID=%s sessionID=%s user=%s model=%s maxWait=%v", requestID, sessionID, userId, modelName, r.sessionBoostMaxWait) - c.AbortWithStatusJSON(http.StatusTooManyRequests, "Request rejected: exceeded maximum session boost queue wait time") + c.AbortWithStatusJSON(http.StatusGatewayTimeout, "Request rejected: exceeded maximum session boost queue wait time") return fmt.Errorf("request rejected: exceeded session boost max queue wait") case <-reqCtx.Done(): // Same admission/abandonment coordination as the waitReject path: Abandon() diff --git a/pkg/kthena-router/router/router_test.go b/pkg/kthena-router/router/router_test.go index d3551c50b..48c48f955 100644 --- a/pkg/kthena-router/router/router_test.go +++ b/pkg/kthena-router/router/router_test.go @@ -1415,7 +1415,7 @@ func TestHandleFairnessScheduling(t *testing.T) { wantHTTPStatus: http.StatusInternalServerError, }, { - name: "session boost wait-reject returns 429", + name: "session boost wait-reject returns 504", fairnessTimeout: 10 * time.Second, setUserID: true, storeWrapper: func(real datastore.Store) datastore.Store { return &blockingEnqueueStore{Store: real} }, @@ -1424,7 +1424,7 @@ func TestHandleFairnessScheduling(t *testing.T) { sessionBoostMaxWait: 50 * time.Millisecond, wantErr: true, wantErrMsg: "exceeded session boost max queue wait", - wantHTTPStatus: http.StatusTooManyRequests, + wantHTTPStatus: http.StatusGatewayTimeout, }, } From 298b2a0fa07e496afc19ff6f2999875b7cb0f58b Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Tue, 7 Jul 2026 08:05:44 +0000 Subject: [PATCH 08/11] docs: regenerate helm-chart-values for session-boost 504 wording Signed-off-by: YaoZengzeng --- docs/kthena/docs/reference/helm-chart-values.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/kthena/docs/reference/helm-chart-values.md b/docs/kthena/docs/reference/helm-chart-values.md index 124c46034..1026d74e6 100644 --- a/docs/kthena/docs/reference/helm-chart-values.md +++ b/docs/kthena/docs/reference/helm-chart-values.md @@ -37,8 +37,8 @@ A Helm chart for deploying Kthena | networking.kthenaRouter.sessionBoost.header | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions. | | networking.kthenaRouter.sessionBoost.inflightPerPod | int | `16` | Maximum inflight requests admitted per backend pod.
Total inflight limit is this value times the number of backend pods.
`0` or unset uses the router default (16). | | networking.kthenaRouter.sessionBoost.maxSessions | int | `4096` | Maximum number of recently-completed sessions kept warm for boosting.
Bounds an LRU cache; the least-recently-used session is evicted automatically.
Size it by the number of concurrent conversations to keep boosted. | -| networking.kthenaRouter.sessionBoost.maxWait | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 429. Only effective when `waitRejectEnabled` is `true`. | -| networking.kthenaRouter.sessionBoost.waitRejectEnabled | bool | `false` | Reject requests that wait longer than `maxWait` in the session-boost
queue with HTTP 429 instead of waiting indefinitely for backend capacity.
Disabled by default. | +| networking.kthenaRouter.sessionBoost.maxWait | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 504. Only effective when `waitRejectEnabled` is `true`. | +| networking.kthenaRouter.sessionBoost.waitRejectEnabled | bool | `false` | Reject requests that wait longer than `maxWait` in the session-boost
queue with HTTP 504 instead of waiting indefinitely for backend capacity.
Disabled by default. | | networking.kthenaRouter.terminationGracePeriodSeconds | int | `330` | The router will drain all in-flight requests before forcefully closing connections. | | networking.kthenaRouter.tls.dnsName | string | `"your-domain.com"` | DNS name to use for the certificate. | | networking.kthenaRouter.tls.enabled | bool | `false` | Enable TLS for Kthena Router server. | From 4e2963ca039022b0792356db1898a1498599c853 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Thu, 9 Jul 2026 02:32:05 +0000 Subject: [PATCH 09/11] session boost: replace wait-reject flag with implicit SESSION_BOOST_TIMEOUT Address hzxuzhonghu's review comments: - Remove SESSION_BOOST_WAIT_REJECT_ENABLED; the timeout is enabled implicitly and controlled solely by SESSION_BOOST_TIMEOUT (default 30s, non-positive disables). - Rename SESSION_BOOST_MAX_WAIT to SESSION_BOOST_TIMEOUT to align with FAIRNESS_QUEUE_TIMEOUT. - Apply the timeout to the request context via context.WithTimeout and reuse the existing reqCtx.Done() case instead of a separate timer/select branch. - Use a strategy-aware log prefix (SessionBoost vs FairnessScheduling) in the shared handler. - Update Helm charts/templates, docs, proposal, and tests accordingly. Signed-off-by: YaoZengzeng --- charts/kthena/charts/networking/README.md | 27 ++-- .../kthena-router/component/deployment.yaml | 8 +- charts/kthena/charts/networking/values.yaml | 13 +- charts/kthena/values.yaml | 9 +- .../docs/reference/helm-chart-values.md | 103 ++++++------ docs/kthena/docs/user-guide/session-boost.md | 46 +++--- docs/proposal/session-boost-strategy.md | 32 ++-- .../datastore/session_boost_queue_test.go | 2 +- pkg/kthena-router/router/router.go | 151 ++++++++---------- pkg/kthena-router/router/router_test.go | 17 +- 10 files changed, 186 insertions(+), 222 deletions(-) diff --git a/charts/kthena/charts/networking/README.md b/charts/kthena/charts/networking/README.md index aabc45c88..30aa3cd63 100644 --- a/charts/kthena/charts/networking/README.md +++ b/charts/kthena/charts/networking/README.md @@ -52,20 +52,19 @@ kthenaRouter: #### Configuration Parameters -| Parameter | Type | Default | Description | -| --------------------------------------------- | ------- | ---------------- | ------------------------------------------------------------------ | -| `kthenaRouter.fairness.enabled` | boolean | `false` | Enable user-fairness scheduling (mutually exclusive with boost) | -| `kthenaRouter.fairness.windowSize` | string | `"1h"` | Fairness: sliding window duration (1m-1h) | -| `kthenaRouter.fairness.inputTokenWeight` | float | `1.0` | Fairness: weight for input tokens (≥0) | -| `kthenaRouter.fairness.outputTokenWeight` | float | `2.0` | Fairness: weight for output tokens (≥0) | -| `kthenaRouter.fairness.maxConcurrent` | int | `0` | Fairness: global inflight limit (`0` falls back to QPS mode) | -| `kthenaRouter.sessionBoost.enabled` | boolean | `false` | Enable session-boost scheduling (mutually exclusive with fairness) | -| `kthenaRouter.sessionBoost.header` | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions | -| `kthenaRouter.sessionBoost.maxSessions` | int | `4096` | Max recently-completed sessions kept warm (LRU-evicted) | -| `kthenaRouter.sessionBoost.inflightPerPod` | int | `16` | Inflight requests per backend pod; total = perPod x pod count | -| `kthenaRouter.sessionBoost.gracePeriod` | string | `"0s"` | Wait time for a same-session follow-up (disabled by default) | -| `kthenaRouter.sessionBoost.waitRejectEnabled` | boolean | `false` | Reject requests waiting longer than `maxWait` with HTTP 504 | -| `kthenaRouter.sessionBoost.maxWait` | string | `"30s"` | Max queue wait before 504 (only when `waitRejectEnabled: true`) | +| Parameter | Type | Default | Description | +| ------------------------------------------ | ------- | ---------------- | ----------------------------------------------------------------------------- | +| `kthenaRouter.fairness.enabled` | boolean | `false` | Enable user-fairness scheduling (mutually exclusive with boost) | +| `kthenaRouter.fairness.windowSize` | string | `"1h"` | Fairness: sliding window duration (1m-1h) | +| `kthenaRouter.fairness.inputTokenWeight` | float | `1.0` | Fairness: weight for input tokens (≥0) | +| `kthenaRouter.fairness.outputTokenWeight` | float | `2.0` | Fairness: weight for output tokens (≥0) | +| `kthenaRouter.fairness.maxConcurrent` | int | `0` | Fairness: global inflight limit (`0` falls back to QPS mode) | +| `kthenaRouter.sessionBoost.enabled` | boolean | `false` | Enable session-boost scheduling (mutually exclusive with fairness) | +| `kthenaRouter.sessionBoost.header` | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions | +| `kthenaRouter.sessionBoost.maxSessions` | int | `4096` | Max recently-completed sessions kept warm (LRU-evicted) | +| `kthenaRouter.sessionBoost.inflightPerPod` | int | `16` | Inflight requests per backend pod; total = perPod x pod count | +| `kthenaRouter.sessionBoost.gracePeriod` | string | `"0s"` | Wait time for a same-session follow-up (disabled by default) | +| `kthenaRouter.sessionBoost.timeout` | string | `"30s"` | Max queue wait before 504; set a non-positive duration (e.g. `0s`) to disable | #### Session Boost Configuration diff --git a/charts/kthena/charts/networking/templates/kthena-router/component/deployment.yaml b/charts/kthena/charts/networking/templates/kthena-router/component/deployment.yaml index b21d4c547..e4087762f 100644 --- a/charts/kthena/charts/networking/templates/kthena-router/component/deployment.yaml +++ b/charts/kthena/charts/networking/templates/kthena-router/component/deployment.yaml @@ -106,11 +106,9 @@ spec: {{- end }} - name: SESSION_BOOST_GRACE_PERIOD value: {{ .gracePeriod | quote }} - {{- if .waitRejectEnabled }} - - name: SESSION_BOOST_WAIT_REJECT_ENABLED - value: "true" - - name: SESSION_BOOST_MAX_WAIT - value: {{ .maxWait | quote }} + {{- if .timeout }} + - name: SESSION_BOOST_TIMEOUT + value: {{ .timeout | quote }} {{- end }} {{- end }} {{- end }} diff --git a/charts/kthena/charts/networking/values.yaml b/charts/kthena/charts/networking/values.yaml index 4318820c0..28ac1edcd 100644 --- a/charts/kthena/charts/networking/values.yaml +++ b/charts/kthena/charts/networking/values.yaml @@ -70,14 +70,11 @@ kthenaRouter: # follow-up to arrive. Disabled by default (0s); enable only if you understand # the latency trade-off for non-boosted requests. gracePeriod: "0s" - # waitRejectEnabled controls whether requests that wait in the session-boost - # queue longer than maxWait are rejected with HTTP 504 instead of waiting - # indefinitely for backend capacity. Disabled by default. - waitRejectEnabled: false - # maxWait is the maximum time a request may wait in the session-boost queue - # before it is rejected with HTTP 504. Only effective when waitRejectEnabled is - # true. - maxWait: "30s" + # timeout is the maximum time a request may wait in the session-boost queue + # before it is rejected with HTTP 504. Defaults to 30s; set it to a non-positive + # duration (e.g. "0s") to disable the timeout, in which case a request is bounded + # only by client disconnect. + timeout: "30s" # accessLog configuration for request logging accessLog: # enabled controls whether access logging is active diff --git a/charts/kthena/values.yaml b/charts/kthena/values.yaml index 9aed836f8..8ba840225 100644 --- a/charts/kthena/values.yaml +++ b/charts/kthena/values.yaml @@ -100,13 +100,10 @@ networking: # -- Wait time after a request completes for a same-session follow-up.
# Disabled by default (`0s`). gracePeriod: "0s" - # -- Reject requests that wait longer than `maxWait` in the session-boost
- # queue with HTTP 504 instead of waiting indefinitely for backend capacity.
- # Disabled by default. - waitRejectEnabled: false # -- Maximum time a request may wait in the session-boost queue before it is
- # rejected with HTTP 504. Only effective when `waitRejectEnabled` is `true`. - maxWait: "30s" + # rejected with HTTP 504. Defaults to `30s`; set a non-positive duration
+ # (e.g. `0s`) to disable it (bounded only by client disconnect). + timeout: "30s" gatewayAPI: # -- Enable Gateway API related features. enabled: false diff --git a/docs/kthena/docs/reference/helm-chart-values.md b/docs/kthena/docs/reference/helm-chart-values.md index 1026d74e6..281e79c3f 100644 --- a/docs/kthena/docs/reference/helm-chart-values.md +++ b/docs/kthena/docs/reference/helm-chart-values.md @@ -6,61 +6,60 @@ A Helm chart for deploying Kthena ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| | networking | 1.0.0 | -| | workload | 1.0.0 | +| Repository | Name | Version | +| ---------- | ---------- | ------- | +| | networking | 1.0.0 | +| | workload | 1.0.0 | ## Values -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| global.certManagementMode | string | `"auto"` | Certificate Management Mode.
Three mutually exclusive options for managing TLS certificates:
- `auto`: Webhook servers generate self-signed certificates automatically.
- `cert-manager`: Use cert-manager to generate and manage certificates (requires cert-manager installation).
- `manual`: Provide your own certificates via caBundle. | -| global.webhook.caBundle | string | `""` | CA bundle for webhook server certificates (base64-encoded).
This is ONLY required when `certManagementMode` is set to "manual".
You can generate it with: `cat /path/to/your/ca.crt | base64 | tr -d '\n'`
| -| networking.enabled | bool | `true` | Enable the networking subchart. | -| networking.kthenaRouter.debugPort | int | `15000` | Debug server port for Kthena Router (localhost only). | -| networking.kthenaRouter.drainTimeout | string | `"5m"` | This should be less than terminationGracePeriodSeconds. | -| networking.kthenaRouter.enabled | bool | `true` | Enable Kthena Router. | -| networking.kthenaRouter.fairness.enabled | bool | `false` | Enable user-fairness scheduling. Mutually exclusive with sessionBoost. | -| networking.kthenaRouter.fairness.inputTokenWeight | float | `1` | User-fairness strategy: weight multiplier for input tokens. | -| networking.kthenaRouter.fairness.maxConcurrent | int | `0` | Global total inflight request limit admitted through the fairness gate.
`0` or unset falls back to QPS-based rate limiting. | -| networking.kthenaRouter.fairness.outputTokenWeight | float | `2` | User-fairness strategy: weight multiplier for output tokens. | -| networking.kthenaRouter.fairness.windowSize | string | `"1h"` | User-fairness strategy: sliding window duration for token usage tracking. | -| networking.kthenaRouter.gatewayAPI.enabled | bool | `false` | Enable Gateway API related features. | -| networking.kthenaRouter.gatewayAPI.inferenceExtension | bool | `false` | Enable Gateway API Inference Extension features.
Requires `gatewayAPI.enabled` to be true. | -| networking.kthenaRouter.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for Kthena Router. | -| networking.kthenaRouter.image.repository | string | `"ghcr.io/volcano-sh/kthena-router"` | Image repository for Kthena Router. | -| networking.kthenaRouter.image.tag | string | `"latest"` | Image tag for Kthena Router. | -| networking.kthenaRouter.port | int | `8080` | Container port for Kthena Router. | -| networking.kthenaRouter.sessionBoost.enabled | bool | `false` | Enable session-boost scheduling. Mutually exclusive with fairness. | -| networking.kthenaRouter.sessionBoost.gracePeriod | string | `"0s"` | Wait time after a request completes for a same-session follow-up.
Disabled by default (`0s`). | -| networking.kthenaRouter.sessionBoost.header | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions. | -| networking.kthenaRouter.sessionBoost.inflightPerPod | int | `16` | Maximum inflight requests admitted per backend pod.
Total inflight limit is this value times the number of backend pods.
`0` or unset uses the router default (16). | -| networking.kthenaRouter.sessionBoost.maxSessions | int | `4096` | Maximum number of recently-completed sessions kept warm for boosting.
Bounds an LRU cache; the least-recently-used session is evicted automatically.
Size it by the number of concurrent conversations to keep boosted. | -| networking.kthenaRouter.sessionBoost.maxWait | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 504. Only effective when `waitRejectEnabled` is `true`. | -| networking.kthenaRouter.sessionBoost.waitRejectEnabled | bool | `false` | Reject requests that wait longer than `maxWait` in the session-boost
queue with HTTP 504 instead of waiting indefinitely for backend capacity.
Disabled by default. | -| networking.kthenaRouter.terminationGracePeriodSeconds | int | `330` | The router will drain all in-flight requests before forcefully closing connections. | -| networking.kthenaRouter.tls.dnsName | string | `"your-domain.com"` | DNS name to use for the certificate. | -| networking.kthenaRouter.tls.enabled | bool | `false` | Enable TLS for Kthena Router server. | -| networking.kthenaRouter.tls.secretName | string | `"kthena-router-tls"` | Secret name to store the certificate and key. | -| networking.kthenaRouter.webhook.enabled | bool | `true` | Enable webhook for Kthena Router. | -| networking.kthenaRouter.webhook.port | int | `8443` | Container port for Kthena Router webhook. | -| networking.kthenaRouter.webhook.servicePort | int | `443` | Service port for Kthena Router webhook. | -| networking.kthenaRouter.webhook.tls.certFile | string | `"/etc/tls/tls.crt"` | Certificate file path for the webhook. | -| networking.kthenaRouter.webhook.tls.keyFile | string | `"/etc/tls/tls.key"` | Key file path for the webhook. | -| networking.kthenaRouter.webhook.tls.secretName | string | `"kthena-router-webhook-certs"` | Secret name for storing webhook certificates. | -| workload.controllerManager.debugPort | int | `0` | Debug server port for Controller Manager (set 0 to disable). | -| workload.controllerManager.downloaderImage.repository | string | `"ghcr.io/volcano-sh/downloader"` | Image repository for the Downloader. | -| workload.controllerManager.downloaderImage.tag | string | `"latest"` | Image tag for the Downloader. | -| workload.controllerManager.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for the Controller Manager. | -| workload.controllerManager.image.repository | string | `"ghcr.io/volcano-sh/kthena-controller-manager"` | Image repository for the Controller Manager. | -| workload.controllerManager.image.tag | string | `"latest"` | Image tag for the Controller Manager. | -| workload.controllerManager.runtimeImage.repository | string | `"ghcr.io/volcano-sh/runtime"` | Image repository for the Runtime. | -| workload.controllerManager.runtimeImage.tag | string | `"latest"` | Image tag for the Runtime. | -| workload.controllerManager.webhook.enabled | bool | `true` | Enable webhook for the Controller Manager. | -| workload.controllerManager.webhook.tls.certSecretName | string | `"kthena-controller-manager-webhook-certs"` | Secret name for storing webhook certificates. | -| workload.controllerManager.webhook.tls.serviceName | string | `"kthena-controller-manager-webhook"` | Service name for the webhook. | -| workload.enabled | bool | `true` | Enable the workload subchart. | +| Key | Type | Default | Description | +| ----------------------------------------------------- | ------ | ------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| global.certManagementMode | string | `"auto"` | Certificate Management Mode.
Three mutually exclusive options for managing TLS certificates:
- `auto`: Webhook servers generate self-signed certificates automatically.
- `cert-manager`: Use cert-manager to generate and manage certificates (requires cert-manager installation).
- `manual`: Provide your own certificates via caBundle. | +| global.webhook.caBundle | string | `""` | CA bundle for webhook server certificates (base64-encoded).
This is ONLY required when `certManagementMode` is set to "manual".
You can generate it with: `cat /path/to/your/ca.crt | base64 | tr -d '\n'`
| +| networking.enabled | bool | `true` | Enable the networking subchart. | +| networking.kthenaRouter.debugPort | int | `15000` | Debug server port for Kthena Router (localhost only). | +| networking.kthenaRouter.drainTimeout | string | `"5m"` | This should be less than terminationGracePeriodSeconds. | +| networking.kthenaRouter.enabled | bool | `true` | Enable Kthena Router. | +| networking.kthenaRouter.fairness.enabled | bool | `false` | Enable user-fairness scheduling. Mutually exclusive with sessionBoost. | +| networking.kthenaRouter.fairness.inputTokenWeight | float | `1` | User-fairness strategy: weight multiplier for input tokens. | +| networking.kthenaRouter.fairness.maxConcurrent | int | `0` | Global total inflight request limit admitted through the fairness gate.
`0` or unset falls back to QPS-based rate limiting. | +| networking.kthenaRouter.fairness.outputTokenWeight | float | `2` | User-fairness strategy: weight multiplier for output tokens. | +| networking.kthenaRouter.fairness.windowSize | string | `"1h"` | User-fairness strategy: sliding window duration for token usage tracking. | +| networking.kthenaRouter.gatewayAPI.enabled | bool | `false` | Enable Gateway API related features. | +| networking.kthenaRouter.gatewayAPI.inferenceExtension | bool | `false` | Enable Gateway API Inference Extension features.
Requires `gatewayAPI.enabled` to be true. | +| networking.kthenaRouter.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for Kthena Router. | +| networking.kthenaRouter.image.repository | string | `"ghcr.io/volcano-sh/kthena-router"` | Image repository for Kthena Router. | +| networking.kthenaRouter.image.tag | string | `"latest"` | Image tag for Kthena Router. | +| networking.kthenaRouter.port | int | `8080` | Container port for Kthena Router. | +| networking.kthenaRouter.sessionBoost.enabled | bool | `false` | Enable session-boost scheduling. Mutually exclusive with fairness. | +| networking.kthenaRouter.sessionBoost.gracePeriod | string | `"0s"` | Wait time after a request completes for a same-session follow-up.
Disabled by default (`0s`). | +| networking.kthenaRouter.sessionBoost.header | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions. | +| networking.kthenaRouter.sessionBoost.inflightPerPod | int | `16` | Maximum inflight requests admitted per backend pod.
Total inflight limit is this value times the number of backend pods.
`0` or unset uses the router default (16). | +| networking.kthenaRouter.sessionBoost.maxSessions | int | `4096` | Maximum number of recently-completed sessions kept warm for boosting.
Bounds an LRU cache; the least-recently-used session is evicted automatically.
Size it by the number of concurrent conversations to keep boosted. | +| networking.kthenaRouter.sessionBoost.timeout | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 504. Defaults to `30s`; set a non-positive duration
(e.g. `0s`) to disable it (bounded only by client disconnect). | +| networking.kthenaRouter.terminationGracePeriodSeconds | int | `330` | The router will drain all in-flight requests before forcefully closing connections. | +| networking.kthenaRouter.tls.dnsName | string | `"your-domain.com"` | DNS name to use for the certificate. | +| networking.kthenaRouter.tls.enabled | bool | `false` | Enable TLS for Kthena Router server. | +| networking.kthenaRouter.tls.secretName | string | `"kthena-router-tls"` | Secret name to store the certificate and key. | +| networking.kthenaRouter.webhook.enabled | bool | `true` | Enable webhook for Kthena Router. | +| networking.kthenaRouter.webhook.port | int | `8443` | Container port for Kthena Router webhook. | +| networking.kthenaRouter.webhook.servicePort | int | `443` | Service port for Kthena Router webhook. | +| networking.kthenaRouter.webhook.tls.certFile | string | `"/etc/tls/tls.crt"` | Certificate file path for the webhook. | +| networking.kthenaRouter.webhook.tls.keyFile | string | `"/etc/tls/tls.key"` | Key file path for the webhook. | +| networking.kthenaRouter.webhook.tls.secretName | string | `"kthena-router-webhook-certs"` | Secret name for storing webhook certificates. | +| workload.controllerManager.debugPort | int | `0` | Debug server port for Controller Manager (set 0 to disable). | +| workload.controllerManager.downloaderImage.repository | string | `"ghcr.io/volcano-sh/downloader"` | Image repository for the Downloader. | +| workload.controllerManager.downloaderImage.tag | string | `"latest"` | Image tag for the Downloader. | +| workload.controllerManager.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for the Controller Manager. | +| workload.controllerManager.image.repository | string | `"ghcr.io/volcano-sh/kthena-controller-manager"` | Image repository for the Controller Manager. | +| workload.controllerManager.image.tag | string | `"latest"` | Image tag for the Controller Manager. | +| workload.controllerManager.runtimeImage.repository | string | `"ghcr.io/volcano-sh/runtime"` | Image repository for the Runtime. | +| workload.controllerManager.runtimeImage.tag | string | `"latest"` | Image tag for the Runtime. | +| workload.controllerManager.webhook.enabled | bool | `true` | Enable webhook for the Controller Manager. | +| workload.controllerManager.webhook.tls.certSecretName | string | `"kthena-controller-manager-webhook-certs"` | Secret name for storing webhook certificates. | +| workload.controllerManager.webhook.tls.serviceName | string | `"kthena-controller-manager-webhook"` | Service name for the webhook. | +| workload.enabled | bool | `true` | Enable the workload subchart. | ## Notes diff --git a/docs/kthena/docs/user-guide/session-boost.md b/docs/kthena/docs/user-guide/session-boost.md index 890dfdd37..d12b2dc44 100644 --- a/docs/kthena/docs/user-guide/session-boost.md +++ b/docs/kthena/docs/user-guide/session-boost.md @@ -30,8 +30,8 @@ flowchart TD E --> F F -- Yes --> G[Dispatch to backend] F -- No --> H[Wait / backpressure] - H --> K{Waited longer than
SESSION_BOOST_MAX_WAIT?} - K -- Yes, wait-reject enabled --> R[Reject with HTTP 504] + H --> K{Waited longer than
SESSION_BOOST_TIMEOUT?} + K -- Yes --> R[Reject with HTTP 504] K -- No --> F G --> I[Request completes] I --> J[Mark session completed
promote in LRU cache] @@ -75,8 +75,7 @@ networking: header: "X-Session-ID" maxSessions: 4096 # LRU cache of recently-completed sessions kept warm inflightPerPod: 16 # total inflight = inflightPerPod x backend pod count - waitRejectEnabled: true # reject requests that wait too long with HTTP 504 - maxWait: "30s" # max queue wait before a 504 is returned + timeout: "30s" # reject requests that wait longer than this with HTTP 504 (set "0s" to disable) ``` Apply with Helm: @@ -102,22 +101,19 @@ env: value: "4096" - name: SESSION_BOOST_INFLIGHT_PER_POD value: "16" # total inflight = perPod x backend pod count -- name: SESSION_BOOST_WAIT_REJECT_ENABLED - value: "true" # reject requests that wait too long with HTTP 504 -- name: SESSION_BOOST_MAX_WAIT - value: "30s" # max queue wait before a 504 is returned +- name: SESSION_BOOST_TIMEOUT + value: "30s" # reject requests that wait longer than this with HTTP 504 (set "0s" to disable) ``` ## Configuration Reference -| Environment Variable | Purpose | Default | Notes | -| ----------------------------------- | -------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `ENABLE_SESSION_BOOST` | Enable the session-boost scheduling strategy | `false` | Mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING` | -| `SESSION_BOOST_HEADER` | HTTP header used to identify conversation sessions | `X-Session-ID` | Must match what your clients send | -| `SESSION_BOOST_MAX_SESSIONS` | Max number of recently-completed sessions kept "warm" for boosting (LRU-bounded) | `4096` | When the cache is full, the least-recently-used session is evicted automatically. Size it by the number of concurrent conversations you want to keep boosted—no time-based tuning required | -| `SESSION_BOOST_INFLIGHT_PER_POD` | Inflight requests admitted per backend pod | `16` | The total inflight limit is this value multiplied by the number of backend pods. Size it from the per-pod concurrency (e.g., vLLM's `--max-num-seqs`) | -| `SESSION_BOOST_WAIT_REJECT_ENABLED` | Reject requests that wait too long in the queue with HTTP 504 | `false` | When `true`, a request queued longer than `SESSION_BOOST_MAX_WAIT` is rejected with `504 Gateway Timeout` instead of continuing to wait | -| `SESSION_BOOST_MAX_WAIT` | Maximum time a request may wait in the queue before it is rejected with 504 | `30s` | Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. This is the only server-side queue-wait bound in session-boost mode; `FAIRNESS_QUEUE_TIMEOUT` does not apply | +| Environment Variable | Purpose | Default | Notes | +| -------------------------------- | -------------------------------------------------------------------------------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ENABLE_SESSION_BOOST` | Enable the session-boost scheduling strategy | `false` | Mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING` | +| `SESSION_BOOST_HEADER` | HTTP header used to identify conversation sessions | `X-Session-ID` | Must match what your clients send | +| `SESSION_BOOST_MAX_SESSIONS` | Max number of recently-completed sessions kept "warm" for boosting (LRU-bounded) | `4096` | When the cache is full, the least-recently-used session is evicted automatically. Size it by the number of concurrent conversations you want to keep boosted—no time-based tuning required | +| `SESSION_BOOST_INFLIGHT_PER_POD` | Inflight requests admitted per backend pod | `16` | The total inflight limit is this value multiplied by the number of backend pods. Size it from the per-pod concurrency (e.g., vLLM's `--max-num-seqs`) | +| `SESSION_BOOST_TIMEOUT` | Maximum time a request may wait in the queue before it is rejected with 504 | `30s` | Enabled by default. A request queued longer than this is rejected with `504 Gateway Timeout`. Set a non-positive duration (e.g. `0s`) to disable it. This is the only server-side queue-wait bound in session-boost mode; `FAIRNESS_QUEUE_TIMEOUT` does not apply | > `SESSION_BOOST_GRACE_PERIOD` is intentionally omitted from the table above. It is an advanced, scenario-specific knob that is disabled by default; see [Advanced: Grace Period](#advanced-grace-period-use-with-caution). @@ -186,16 +182,16 @@ When a request completes, the queue immediately attempts to dequeue the next req ### Queue Wait Timeout (504 Rejection) -Under heavy load a request may sit in the queue for a long time before backend capacity frees up. Session boost does **not** apply the fairness queue timeout (`FAIRNESS_QUEUE_TIMEOUT` only governs the user-fairness queue); by default a queued request in session-boost mode waits until backend capacity frees up or the client disconnects. For latency-sensitive front ends it is often better to **fail fast** and let the client retry or shed load. +Under heavy load a request may sit in the queue for a long time before backend capacity frees up. Session boost does **not** apply the fairness queue timeout (`FAIRNESS_QUEUE_TIMEOUT` only governs the user-fairness queue). Instead, session boost bounds the wait with its own **queue wait timeout** so latency-sensitive front ends **fail fast** and can retry or shed load rather than waiting indefinitely. -Session boost provides its own optional **wait timeout** that rejects over-queued requests early with `504 Gateway Timeout`: +The wait timeout rejects over-queued requests with `504 Gateway Timeout`: -- Enable it with `SESSION_BOOST_WAIT_REJECT_ENABLED=true`. -- Set the threshold with `SESSION_BOOST_MAX_WAIT` (default `30s`). +- It is controlled by `SESSION_BOOST_TIMEOUT` and **enabled by default** at `30s`. +- Set a non-positive duration (e.g. `0s`) to disable it; a session-boost request is then bounded only by client disconnect. -When enabled, if a request waits in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router removes it from the queue and responds with `504 Gateway Timeout`. The 504 signals that the request *timed out* waiting for backend capacity (the queue is saturated); the client can retry later or shed load. +If a request waits in the queue longer than `SESSION_BOOST_TIMEOUT`, the router removes it from the queue and responds with `504 Gateway Timeout`. The 504 signals that the request *timed out* waiting for backend capacity (the queue is saturated); the client can retry later or shed load. -> `SESSION_BOOST_MAX_WAIT` is the only server-side queue-wait bound in session-boost mode. `FAIRNESS_QUEUE_TIMEOUT` has no effect here—it applies exclusively to the user-fairness strategy. Without `SESSION_BOOST_WAIT_REJECT_ENABLED`, a session-boost request is bounded only by client disconnect. +> `SESSION_BOOST_TIMEOUT` is the only server-side queue-wait bound in session-boost mode. `FAIRNESS_QUEUE_TIMEOUT` has no effect here—it applies exclusively to the user-fairness strategy. When `SESSION_BOOST_TIMEOUT` is disabled (a non-positive value), a session-boost request is bounded only by client disconnect. ## Session Boost vs User Fairness @@ -274,11 +270,11 @@ Each tracked session consumes minimal memory (just a session ID). The tracker is ### Clients receive HTTP 504 responses -When `SESSION_BOOST_WAIT_REJECT_ENABLED=true`, the router returns `504 Gateway Timeout` for requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT`. This is expected load-shedding under sustained overload—it means the backends cannot keep up with demand. If you see more 504s than desired: +When `SESSION_BOOST_TIMEOUT` is set (enabled by default at `30s`), the router returns `504 Gateway Timeout` for requests that wait in the queue longer than it. This is expected load-shedding under sustained overload—it means the backends cannot keep up with demand. If you see more 504s than desired: 1. Confirm the backends are healthy and scaled appropriately; add capacity if they are saturated. -2. Increase `SESSION_BOOST_MAX_WAIT` to allow requests to wait longer before rejection. -3. Disable the feature (`SESSION_BOOST_WAIT_REJECT_ENABLED=false`) so requests are no longer rejected on a wait timeout (they then wait until backend capacity frees up or the client disconnects). +2. Increase `SESSION_BOOST_TIMEOUT` to allow requests to wait longer before rejection. +3. Disable the timeout by setting `SESSION_BOOST_TIMEOUT` to a non-positive value (e.g. `0s`) so requests are no longer rejected on a wait timeout (they then wait until backend capacity frees up or the client disconnects). ## Advanced: Grace Period (Use With Caution) diff --git a/docs/proposal/session-boost-strategy.md b/docs/proposal/session-boost-strategy.md index 8126f7377..05b9b12eb 100644 --- a/docs/proposal/session-boost-strategy.md +++ b/docs/proposal/session-boost-strategy.md @@ -219,15 +219,14 @@ The net effect is a strict precedence: **grace timing → inflight gate → back #### Configuration -| Environment Variable | Default | Description | -| ----------------------------------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ENABLE_SESSION_BOOST` | `false` | Enable the session-boost scheduling strategy (mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING`) | -| `SESSION_BOOST_HEADER` | `X-Session-ID` | HTTP header used to identify conversation sessions | -| `SESSION_BOOST_MAX_SESSIONS` | `4096` | Maximum number of recently-completed sessions kept warm for boosting. Bounds an LRU cache; the least-recently-used session is evicted when exceeded. Sized by session count, not time | -| `SESSION_BOOST_GRACE_PERIOD` | `0` | Wait time after release for same-session follow-up. Disabled by default; enable only when you understand the latency trade-off | -| `SESSION_BOOST_INFLIGHT_PER_POD` | `16` | Inflight requests admitted per backend pod; total inflight = perPod x backend pod count. Size it from the estimated per-pod concurrency (e.g. vLLM's --max-num-seqs) | -| `SESSION_BOOST_WAIT_REJECT_ENABLED` | `false` | Reject requests that wait in the queue longer than `SESSION_BOOST_MAX_WAIT` with HTTP 504. `FAIRNESS_QUEUE_TIMEOUT` does not apply in session-boost mode | -| `SESSION_BOOST_MAX_WAIT` | `30s` | Maximum time a request may wait in the queue before it is rejected with HTTP 504. Only effective when `SESSION_BOOST_WAIT_REJECT_ENABLED=true`; the only server-side queue-wait bound in session-boost mode | +| Environment Variable | Default | Description | +| -------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ENABLE_SESSION_BOOST` | `false` | Enable the session-boost scheduling strategy (mutually exclusive with `ENABLE_FAIRNESS_SCHEDULING`) | +| `SESSION_BOOST_HEADER` | `X-Session-ID` | HTTP header used to identify conversation sessions | +| `SESSION_BOOST_MAX_SESSIONS` | `4096` | Maximum number of recently-completed sessions kept warm for boosting. Bounds an LRU cache; the least-recently-used session is evicted when exceeded. Sized by session count, not time | +| `SESSION_BOOST_GRACE_PERIOD` | `0` | Wait time after release for same-session follow-up. Disabled by default; enable only when you understand the latency trade-off | +| `SESSION_BOOST_INFLIGHT_PER_POD` | `16` | Inflight requests admitted per backend pod; total inflight = perPod x backend pod count. Size it from the estimated per-pod concurrency (e.g. vLLM's --max-num-seqs) | +| `SESSION_BOOST_TIMEOUT` | `30s` | Maximum time a request may wait in the queue before it is rejected with HTTP 504. Enabled by default; set a non-positive duration (e.g. `0s`) to disable it. It is the only server-side queue-wait bound in session-boost mode (`FAIRNESS_QUEUE_TIMEOUT` does not apply) | ### Design Details @@ -302,21 +301,20 @@ When a request completes (Release), the queue immediately attempts to dequeue th #### Queue Wait Timeout (504 Rejection) -Under sustained overload the two-level admission control legitimately holds requests in the queue until backend capacity frees up. `FAIRNESS_QUEUE_TIMEOUT` governs **only** the user-fairness queue and does not apply in session-boost mode, so by default a queued session-boost request waits until backend capacity frees up or the client disconnects (which returns `503`). For latency-sensitive front ends this open-ended wait can be undesirable: the client would rather be told *quickly* that the system is saturated so it can retry elsewhere or shed load. +Under sustained overload the two-level admission control legitimately holds requests in the queue until backend capacity frees up. `FAIRNESS_QUEUE_TIMEOUT` governs **only** the user-fairness queue and does not apply in session-boost mode. Instead, session boost bounds the wait with its own timeout so latency-sensitive front ends are told *quickly* that the system is saturated and can retry elsewhere or shed load rather than waiting open-endedly (until the client disconnects, which returns `503`). -The optional queue wait timeout provides this fail-fast behavior: +The queue wait timeout provides this fail-fast behavior: -- It is enabled with `SESSION_BOOST_WAIT_REJECT_ENABLED=true` and disabled by default. -- The threshold is configured with `SESSION_BOOST_MAX_WAIT` (default `30s`). -- When a request has been waiting in the queue longer than `SESSION_BOOST_MAX_WAIT`, the router stops waiting, removes the request from the queue, and responds with `504 Gateway Timeout`. +- It is controlled by `SESSION_BOOST_TIMEOUT` and enabled by default at `30s`. Setting it to a non-positive duration (e.g. `0s`) disables the timeout, leaving a session-boost request bounded only by client disconnect. +- When a request has been waiting in the queue longer than `SESSION_BOOST_TIMEOUT`, the router stops waiting, removes the request from the queue, and responds with `504 Gateway Timeout`. -It is implemented in the router's request-handling path rather than in the queue's dequeue loop, mirroring how the fairness queue's own `FAIRNESS_QUEUE_TIMEOUT` (504) is handled. Both queue-wait timeouts therefore return the same `504` status; they differ only in what arms them (`FAIRNESS_QUEUE_TIMEOUT` vs `SESSION_BOOST_MAX_WAIT`). In session-boost mode the request context is created **without** the fairness deadline, so `FAIRNESS_QUEUE_TIMEOUT` has no effect. When the feature is on, the handler arms a timer for `SESSION_BOOST_MAX_WAIT` alongside the admission (`NotifyChan`) and cancellation (`reqCtx.Done()`) signals it already waits on: +It is implemented in the router's request-handling path rather than in the queue's dequeue loop, mirroring how the fairness queue's own `FAIRNESS_QUEUE_TIMEOUT` (504) is handled. Both queue-wait timeouts therefore return the same `504` status; they differ only in what arms them (`FAIRNESS_QUEUE_TIMEOUT` vs `SESSION_BOOST_TIMEOUT`). In session-boost mode the request context carries the `SESSION_BOOST_TIMEOUT` deadline (via `context.WithTimeout`) instead of the fairness deadline, so `FAIRNESS_QUEUE_TIMEOUT` has no effect. The handler waits on the admission (`NotifyChan`) and cancellation/timeout (`reqCtx.Done()`) signals: - **Admitted first**: the request proceeds to the backend as usual (no rejection). -- **Wait timer fires first**: the handler cancels the request context — which makes the queue drop the request from the heap via its existing cancellation check (`isCancelled` / `drainCancelledLocked`), reusing the same cleanup path as client disconnects — releases any permit that may have been granted concurrently, and returns `504 Gateway Timeout`. +- **Timeout fires first**: the request context's deadline is exceeded, which makes the queue drop the request from the heap via its existing cancellation check (`isCancelled` / `drainCancelledLocked`), reusing the same cleanup path as client disconnects; the handler abandons the request, releases any permit that may have been granted concurrently, and returns `504 Gateway Timeout`. - **Client disconnect fires first**: the existing behavior is unchanged (`503`). -Because it reuses the queue's cancellation cleanup, the wait timeout adds no new state to the queue and no extra polling. It is orthogonal to the inflight and backend capacity gates: those gates decide *whether* a request may run, while the wait timeout only bounds *how long* a request is willing to wait before giving up. `SESSION_BOOST_MAX_WAIT` is the only server-side queue-wait bound in session-boost mode; the feature only applies in session-boost mode. +Because it reuses the queue's cancellation cleanup, the wait timeout adds no new state to the queue and no extra polling. It is orthogonal to the inflight and backend capacity gates: those gates decide *whether* a request may run, while the wait timeout only bounds *how long* a request is willing to wait before giving up. `SESSION_BOOST_TIMEOUT` is the only server-side queue-wait bound in session-boost mode; the feature only applies in session-boost mode. ### Multi-Turn Conversation Advantages diff --git a/pkg/kthena-router/datastore/session_boost_queue_test.go b/pkg/kthena-router/datastore/session_boost_queue_test.go index bd52f75d4..70afc2353 100644 --- a/pkg/kthena-router/datastore/session_boost_queue_test.go +++ b/pkg/kthena-router/datastore/session_boost_queue_test.go @@ -656,7 +656,7 @@ func TestSessionBoost_ConcurrentAdmitAbandonNoLeak(t *testing.T) { q.admitSessionBoost(req) }() // Handler side: time out and abandon; release if admission raced in. This - // mirrors the router's waitReject/timeout handling. + // mirrors the router's queue-wait timeout handling. go func() { defer wg.Done() if req.Abandon() { diff --git a/pkg/kthena-router/router/router.go b/pkg/kthena-router/router/router.go index 90af2978e..cea5326fa 100644 --- a/pkg/kthena-router/router/router.go +++ b/pkg/kthena-router/router/router.go @@ -93,12 +93,11 @@ type Router struct { tokenWeight float64 // Weight for token-based priority in the fairness strategy (default 1.0) requestNumWeight float64 // Weight for request-count-based priority in the fairness strategy (default 0.0) - // Session-boost wait-reject configuration. When sessionBoostWaitRejectEnabled - // is true, a request that waits in the session-boost queue longer than - // sessionBoostMaxWait is rejected with HTTP 504 instead of continuing to wait - // for the general queue timeout. - sessionBoostWaitRejectEnabled bool - sessionBoostMaxWait time.Duration + // Session-boost queue-wait timeout. A request that waits in the session-boost + // queue longer than sessionBoostTimeout is rejected with HTTP 504 instead of + // waiting indefinitely for backend capacity. It defaults to 30s; a non-positive + // value disables the timeout (the request is bounded only by client disconnect). + sessionBoostTimeout time.Duration } // ActiveRequestCount returns the number of requests currently being handled by the router. @@ -190,8 +189,7 @@ func NewRouter(store datastore.Store, routerConfigPath string) *Router { tokenWeight: parseEnvFloat("FAIRNESS_PRIORITY_TOKEN_WEIGHT", 1.0), requestNumWeight: parseEnvFloat("FAIRNESS_PRIORITY_REQUEST_NUM_WEIGHT", 0.0), - sessionBoostWaitRejectEnabled: getEnvBool("SESSION_BOOST_WAIT_REJECT_ENABLED", false), - sessionBoostMaxWait: parseSessionBoostMaxWait(), + sessionBoostTimeout: parseSessionBoostTimeout(), } } @@ -207,27 +205,27 @@ func parseQueueTimeout() time.Duration { return defaultQueueTimeout } -// defaultSessionBoostMaxWait is the maximum time a request may wait in the -// session-boost queue before it is rejected with HTTP 504, used when -// SESSION_BOOST_MAX_WAIT is not set or invalid and wait-reject is enabled. -const defaultSessionBoostMaxWait = 30 * time.Second - -// parseSessionBoostMaxWait reads the session-boost max queue wait from the -// SESSION_BOOST_MAX_WAIT environment variable. It only takes effect when -// SESSION_BOOST_WAIT_REJECT_ENABLED is true, so when wait-reject is disabled we -// skip parsing entirely to avoid emitting confusing warnings for a value that -// has no effect. -func parseSessionBoostMaxWait() time.Duration { - if !getEnvBool("SESSION_BOOST_WAIT_REJECT_ENABLED", false) { - return defaultSessionBoostMaxWait - } - if s, ok := os.LookupEnv("SESSION_BOOST_MAX_WAIT"); ok { - if d, err := time.ParseDuration(s); err == nil && d > 0 { +// defaultSessionBoostTimeout is the session-boost queue-wait timeout applied +// when SESSION_BOOST_TIMEOUT is not set. It is enabled by default so that a +// session-boost request does not wait indefinitely for backend capacity. +const defaultSessionBoostTimeout = 30 * time.Second + +// parseSessionBoostTimeout reads the session-boost queue-wait timeout from the +// SESSION_BOOST_TIMEOUT environment variable. A request that waits in the +// session-boost queue longer than the timeout is rejected with HTTP 504. The +// timeout defaults to defaultSessionBoostTimeout (30s) when the variable is +// unset. Setting it to a non-positive duration (e.g. "0s") disables the timeout, +// in which case a session-boost request is bounded only by client disconnect. An +// invalid value falls back to the default. +func parseSessionBoostTimeout() time.Duration { + if s, ok := os.LookupEnv("SESSION_BOOST_TIMEOUT"); ok { + if d, err := time.ParseDuration(s); err == nil { + // A non-positive duration explicitly disables the timeout. return d } - klog.Warningf("Invalid SESSION_BOOST_MAX_WAIT %q, using default %v", s, defaultSessionBoostMaxWait) + klog.Warningf("Invalid SESSION_BOOST_TIMEOUT %q, using default %v", s, defaultSessionBoostTimeout) } - return defaultSessionBoostMaxWait + return defaultSessionBoostTimeout } func parseEnvFloat(key string, fallback float64) float64 { @@ -1110,20 +1108,32 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ klog.Warningf("user ID not found in request %s", requestID) } - klog.V(4).Infof("[FairnessScheduling] incoming request: reqID=%s user=%s model=%s", - requestID, userId, modelName) + // logPrefix reflects the active scheduling strategy so log lines emitted from + // this shared handler are attributed to the right queue (session boost vs + // user fairness). + logPrefix := "[FairnessScheduling]" + if EnableSessionBoost { + logPrefix = "[SessionBoost]" + } + + klog.V(4).Infof("%s incoming request: reqID=%s user=%s model=%s", + logPrefix, requestID, userId, modelName) // Create the request-scoped context that also drives the queue's cancellation - // cleanup (CancelCh). The general queue-wait deadline differs by strategy: + // cleanup (CancelCh). The queue-wait deadline differs by strategy: // - Fairness mode: bounded by FAIRNESS_QUEUE_TIMEOUT; exceeding it returns 504. - // - Session-boost mode: FAIRNESS_QUEUE_TIMEOUT does NOT apply. Session boost has - // its own independent wait control via SESSION_BOOST_MAX_WAIT (returns 504 when - // SESSION_BOOST_WAIT_REJECT_ENABLED is set); otherwise the request is bounded - // only by client disconnect. + // - Session-boost mode: FAIRNESS_QUEUE_TIMEOUT does NOT apply. SESSION_BOOST_TIMEOUT + // (default 30s) bounds the wait and exceeding it returns 504; setting it to a + // non-positive value disables the timeout, leaving the request bounded only by + // client disconnect. var reqCtx context.Context var cancel context.CancelFunc if EnableSessionBoost { - reqCtx, cancel = context.WithCancel(c.Request.Context()) + if r.sessionBoostTimeout > 0 { + reqCtx, cancel = context.WithTimeout(c.Request.Context(), r.sessionBoostTimeout) + } else { + reqCtx, cancel = context.WithCancel(c.Request.Context()) + } } else { reqCtx, cancel = context.WithTimeout(c.Request.Context(), r.queueTimeout) } @@ -1152,32 +1162,19 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ } if err := r.store.Enqueue(queueReq); err != nil { - klog.Errorf("[FairnessScheduling] failed to enqueue: reqID=%s sessionID=%s user=%s model=%s err=%v", - requestID, sessionID, userId, modelName, err) + klog.Errorf("%s failed to enqueue: reqID=%s sessionID=%s user=%s model=%s err=%v", + logPrefix, requestID, sessionID, userId, modelName, err) c.AbortWithStatusJSON(http.StatusInternalServerError, fmt.Sprintf("failed to enqueue request: %v", err)) return fmt.Errorf("failed to enqueue request: %v", err) } - // Session-boost wait-reject: when enabled, a request that waits in the queue - // longer than sessionBoostMaxWait is rejected with HTTP 504 instead of waiting - // indefinitely for backend capacity (session-boost mode has no - // FAIRNESS_QUEUE_TIMEOUT deadline, so this wait-reject timer is the only - // queue-wait bound). The timer only arms in session-boost mode when the feature - // is enabled and a positive max wait is set. - var waitRejectCh <-chan time.Time - if EnableSessionBoost && r.sessionBoostWaitRejectEnabled && r.sessionBoostMaxWait > 0 { - waitRejectTimer := time.NewTimer(r.sessionBoostMaxWait) - defer waitRejectTimer.Stop() - waitRejectCh = waitRejectTimer.C - } - select { case <-queueReq.NotifyChan: if queueReq.Release != nil { defer queueReq.Release() } - klog.V(4).Infof("[FairnessScheduling] request dequeued: reqID=%s user=%s model=%s sessionBoost=%v waitTime=%v", - requestID, userId, modelName, queueReq.SessionBoost, time.Since(queueReq.RequestTime)) + klog.V(4).Infof("%s request dequeued: reqID=%s user=%s model=%s sessionBoost=%v waitTime=%v", + logPrefix, requestID, userId, modelName, queueReq.SessionBoost, time.Since(queueReq.RequestTime)) lbErr := r.doLoadbalance(c, modelRequest) // After a successful proxy, mark the session request as completed so follow-up @@ -1187,45 +1184,33 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ r.store.MarkSessionRequestCompleted(modelName, sessionID) } return nil - case <-waitRejectCh: - // Exceeded the session-boost maximum queue wait. Cancel the request context - // so the queue drops it from the heap (via its cancellation check), then - // abandon it to reject the client with 504. - cancel() + case <-reqCtx.Done(): // Abandon() atomically coordinates with the dequeue loop: if admission raced // in first it returns true and we own the inflight permit, so release it here; // otherwise it marks the request abandoned so the loop skips admission and no - // permit can leak. This replaces the previous non-blocking NotifyChan receive, - // which could miss an admission that had passed its cancellation check but not - // yet closed NotifyChan, leaking the permit. - if queueReq.Abandon() { - queueReq.Release() - } - // A session-boost wait timeout is expected load-shedding when wait-reject is - // enabled, and under sustained overload this path can fire for many requests, so - // log at a verbose level to avoid flooding the logs (the fairness 504 timeout, - // by contrast, is unexpected and logs at error level). - klog.V(2).Infof("[SessionBoost] request rejected after exceeding max queue wait: reqID=%s sessionID=%s user=%s model=%s maxWait=%v", - requestID, sessionID, userId, modelName, r.sessionBoostMaxWait) - c.AbortWithStatusJSON(http.StatusGatewayTimeout, "Request rejected: exceeded maximum session boost queue wait time") - return fmt.Errorf("request rejected: exceeded session boost max queue wait") - case <-reqCtx.Done(): - // Same admission/abandonment coordination as the waitReject path: Abandon() - // either observes a concurrent admission (returning true so we release the - // permit) or blocks admission, so a timeout/cancel can never leak an inflight - // permit. + // permit can leak. if queueReq.Abandon() { queueReq.Release() } if errors.Is(reqCtx.Err(), context.DeadlineExceeded) { - klog.Errorf("[FairnessScheduling] request timed out in queue: reqID=%s sessionID=%s user=%s model=%s timeout=%v", - requestID, sessionID, userId, modelName, r.queueTimeout) - c.AbortWithStatusJSON(http.StatusGatewayTimeout, "Request processing timed out") - return fmt.Errorf("request processing timed out in fairness queue") + // Exceeded the queue-wait timeout. In session-boost mode this is expected + // load-shedding when SESSION_BOOST_TIMEOUT is set, and under sustained + // overload it can fire for many requests, so log at a verbose level to avoid + // flooding the logs. In fairness mode the FAIRNESS_QUEUE_TIMEOUT is unexpected + // and logs at error level. + if EnableSessionBoost { + klog.V(2).Infof("%s request rejected after exceeding queue-wait timeout: reqID=%s sessionID=%s user=%s model=%s timeout=%v", + logPrefix, requestID, sessionID, userId, modelName, r.sessionBoostTimeout) + } else { + klog.Errorf("%s request timed out in queue: reqID=%s sessionID=%s user=%s model=%s timeout=%v", + logPrefix, requestID, sessionID, userId, modelName, r.queueTimeout) + } + c.AbortWithStatusJSON(http.StatusGatewayTimeout, "Request processing timed out in queue") + return fmt.Errorf("request processing timed out in queue") } - klog.V(4).Infof("[FairnessScheduling] request cancelled (client disconnected): reqID=%s sessionID=%s user=%s model=%s", - requestID, sessionID, userId, modelName) - c.AbortWithStatusJSON(http.StatusServiceUnavailable, "Client disconnected while waiting in fairness queue") - return fmt.Errorf("client disconnected while waiting in fairness queue") + klog.V(4).Infof("%s request cancelled (client disconnected): reqID=%s sessionID=%s user=%s model=%s", + logPrefix, requestID, sessionID, userId, modelName) + c.AbortWithStatusJSON(http.StatusServiceUnavailable, "Client disconnected while waiting in queue") + return fmt.Errorf("client disconnected while waiting in queue") } } diff --git a/pkg/kthena-router/router/router_test.go b/pkg/kthena-router/router/router_test.go index 48c48f955..d9fb50f81 100644 --- a/pkg/kthena-router/router/router_test.go +++ b/pkg/kthena-router/router/router_test.go @@ -1363,10 +1363,9 @@ func TestHandleFairnessScheduling(t *testing.T) { wantErrMsg string wantHTTPStatus int wantBodyContains string - // Session-boost wait-reject configuration for the test case. + // Session-boost queue-wait timeout configuration for the test case. enableSessionBoost bool - waitRejectEnabled bool - sessionBoostMaxWait time.Duration + sessionBoostTimeout time.Duration }{ { name: "happy path with userId", @@ -1415,15 +1414,14 @@ func TestHandleFairnessScheduling(t *testing.T) { wantHTTPStatus: http.StatusInternalServerError, }, { - name: "session boost wait-reject returns 504", + name: "session boost queue-wait timeout returns 504", fairnessTimeout: 10 * time.Second, setUserID: true, storeWrapper: func(real datastore.Store) datastore.Store { return &blockingEnqueueStore{Store: real} }, enableSessionBoost: true, - waitRejectEnabled: true, - sessionBoostMaxWait: 50 * time.Millisecond, + sessionBoostTimeout: 50 * time.Millisecond, wantErr: true, - wantErrMsg: "exceeded session boost max queue wait", + wantErrMsg: "timed out", wantHTTPStatus: http.StatusGatewayTimeout, }, } @@ -1434,10 +1432,7 @@ func TestHandleFairnessScheduling(t *testing.T) { defer backend.Close() router.queueTimeout = tt.fairnessTimeout - router.sessionBoostWaitRejectEnabled = tt.waitRejectEnabled - if tt.sessionBoostMaxWait > 0 { - router.sessionBoostMaxWait = tt.sessionBoostMaxWait - } + router.sessionBoostTimeout = tt.sessionBoostTimeout // Set the package-level flag explicitly for every case (and restore it) // so subtests stay isolated regardless of execution order. prevEnableSessionBoost := EnableSessionBoost From 99c1d4512e3c811a68cbf7a14042694366c484b9 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Thu, 9 Jul 2026 02:39:41 +0000 Subject: [PATCH 10/11] docs: regenerate helm-chart-values with make gen-docs format Signed-off-by: YaoZengzeng --- .../docs/reference/helm-chart-values.md | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/docs/kthena/docs/reference/helm-chart-values.md b/docs/kthena/docs/reference/helm-chart-values.md index 281e79c3f..abc3935f6 100644 --- a/docs/kthena/docs/reference/helm-chart-values.md +++ b/docs/kthena/docs/reference/helm-chart-values.md @@ -6,60 +6,60 @@ A Helm chart for deploying Kthena ## Requirements -| Repository | Name | Version | -| ---------- | ---------- | ------- | -| | networking | 1.0.0 | -| | workload | 1.0.0 | +| Repository | Name | Version | +|------------|------|---------| +| | networking | 1.0.0 | +| | workload | 1.0.0 | ## Values -| Key | Type | Default | Description | -| ----------------------------------------------------- | ------ | ------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| global.certManagementMode | string | `"auto"` | Certificate Management Mode.
Three mutually exclusive options for managing TLS certificates:
- `auto`: Webhook servers generate self-signed certificates automatically.
- `cert-manager`: Use cert-manager to generate and manage certificates (requires cert-manager installation).
- `manual`: Provide your own certificates via caBundle. | -| global.webhook.caBundle | string | `""` | CA bundle for webhook server certificates (base64-encoded).
This is ONLY required when `certManagementMode` is set to "manual".
You can generate it with: `cat /path/to/your/ca.crt | base64 | tr -d '\n'`
| -| networking.enabled | bool | `true` | Enable the networking subchart. | -| networking.kthenaRouter.debugPort | int | `15000` | Debug server port for Kthena Router (localhost only). | -| networking.kthenaRouter.drainTimeout | string | `"5m"` | This should be less than terminationGracePeriodSeconds. | -| networking.kthenaRouter.enabled | bool | `true` | Enable Kthena Router. | -| networking.kthenaRouter.fairness.enabled | bool | `false` | Enable user-fairness scheduling. Mutually exclusive with sessionBoost. | -| networking.kthenaRouter.fairness.inputTokenWeight | float | `1` | User-fairness strategy: weight multiplier for input tokens. | -| networking.kthenaRouter.fairness.maxConcurrent | int | `0` | Global total inflight request limit admitted through the fairness gate.
`0` or unset falls back to QPS-based rate limiting. | -| networking.kthenaRouter.fairness.outputTokenWeight | float | `2` | User-fairness strategy: weight multiplier for output tokens. | -| networking.kthenaRouter.fairness.windowSize | string | `"1h"` | User-fairness strategy: sliding window duration for token usage tracking. | -| networking.kthenaRouter.gatewayAPI.enabled | bool | `false` | Enable Gateway API related features. | -| networking.kthenaRouter.gatewayAPI.inferenceExtension | bool | `false` | Enable Gateway API Inference Extension features.
Requires `gatewayAPI.enabled` to be true. | -| networking.kthenaRouter.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for Kthena Router. | -| networking.kthenaRouter.image.repository | string | `"ghcr.io/volcano-sh/kthena-router"` | Image repository for Kthena Router. | -| networking.kthenaRouter.image.tag | string | `"latest"` | Image tag for Kthena Router. | -| networking.kthenaRouter.port | int | `8080` | Container port for Kthena Router. | -| networking.kthenaRouter.sessionBoost.enabled | bool | `false` | Enable session-boost scheduling. Mutually exclusive with fairness. | -| networking.kthenaRouter.sessionBoost.gracePeriod | string | `"0s"` | Wait time after a request completes for a same-session follow-up.
Disabled by default (`0s`). | -| networking.kthenaRouter.sessionBoost.header | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions. | -| networking.kthenaRouter.sessionBoost.inflightPerPod | int | `16` | Maximum inflight requests admitted per backend pod.
Total inflight limit is this value times the number of backend pods.
`0` or unset uses the router default (16). | -| networking.kthenaRouter.sessionBoost.maxSessions | int | `4096` | Maximum number of recently-completed sessions kept warm for boosting.
Bounds an LRU cache; the least-recently-used session is evicted automatically.
Size it by the number of concurrent conversations to keep boosted. | -| networking.kthenaRouter.sessionBoost.timeout | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 504. Defaults to `30s`; set a non-positive duration
(e.g. `0s`) to disable it (bounded only by client disconnect). | -| networking.kthenaRouter.terminationGracePeriodSeconds | int | `330` | The router will drain all in-flight requests before forcefully closing connections. | -| networking.kthenaRouter.tls.dnsName | string | `"your-domain.com"` | DNS name to use for the certificate. | -| networking.kthenaRouter.tls.enabled | bool | `false` | Enable TLS for Kthena Router server. | -| networking.kthenaRouter.tls.secretName | string | `"kthena-router-tls"` | Secret name to store the certificate and key. | -| networking.kthenaRouter.webhook.enabled | bool | `true` | Enable webhook for Kthena Router. | -| networking.kthenaRouter.webhook.port | int | `8443` | Container port for Kthena Router webhook. | -| networking.kthenaRouter.webhook.servicePort | int | `443` | Service port for Kthena Router webhook. | -| networking.kthenaRouter.webhook.tls.certFile | string | `"/etc/tls/tls.crt"` | Certificate file path for the webhook. | -| networking.kthenaRouter.webhook.tls.keyFile | string | `"/etc/tls/tls.key"` | Key file path for the webhook. | -| networking.kthenaRouter.webhook.tls.secretName | string | `"kthena-router-webhook-certs"` | Secret name for storing webhook certificates. | -| workload.controllerManager.debugPort | int | `0` | Debug server port for Controller Manager (set 0 to disable). | -| workload.controllerManager.downloaderImage.repository | string | `"ghcr.io/volcano-sh/downloader"` | Image repository for the Downloader. | -| workload.controllerManager.downloaderImage.tag | string | `"latest"` | Image tag for the Downloader. | -| workload.controllerManager.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for the Controller Manager. | -| workload.controllerManager.image.repository | string | `"ghcr.io/volcano-sh/kthena-controller-manager"` | Image repository for the Controller Manager. | -| workload.controllerManager.image.tag | string | `"latest"` | Image tag for the Controller Manager. | -| workload.controllerManager.runtimeImage.repository | string | `"ghcr.io/volcano-sh/runtime"` | Image repository for the Runtime. | -| workload.controllerManager.runtimeImage.tag | string | `"latest"` | Image tag for the Runtime. | -| workload.controllerManager.webhook.enabled | bool | `true` | Enable webhook for the Controller Manager. | -| workload.controllerManager.webhook.tls.certSecretName | string | `"kthena-controller-manager-webhook-certs"` | Secret name for storing webhook certificates. | -| workload.controllerManager.webhook.tls.serviceName | string | `"kthena-controller-manager-webhook"` | Service name for the webhook. | -| workload.enabled | bool | `true` | Enable the workload subchart. | +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| global.certManagementMode | string | `"auto"` | Certificate Management Mode.
Three mutually exclusive options for managing TLS certificates:
- `auto`: Webhook servers generate self-signed certificates automatically.
- `cert-manager`: Use cert-manager to generate and manage certificates (requires cert-manager installation).
- `manual`: Provide your own certificates via caBundle. | +| global.webhook.caBundle | string | `""` | CA bundle for webhook server certificates (base64-encoded).
This is ONLY required when `certManagementMode` is set to "manual".
You can generate it with: `cat /path/to/your/ca.crt | base64 | tr -d '\n'`
| +| networking.enabled | bool | `true` | Enable the networking subchart. | +| networking.kthenaRouter.debugPort | int | `15000` | Debug server port for Kthena Router (localhost only). | +| networking.kthenaRouter.drainTimeout | string | `"5m"` | This should be less than terminationGracePeriodSeconds. | +| networking.kthenaRouter.enabled | bool | `true` | Enable Kthena Router. | +| networking.kthenaRouter.fairness.enabled | bool | `false` | Enable user-fairness scheduling. Mutually exclusive with sessionBoost. | +| networking.kthenaRouter.fairness.inputTokenWeight | float | `1` | User-fairness strategy: weight multiplier for input tokens. | +| networking.kthenaRouter.fairness.maxConcurrent | int | `0` | Global total inflight request limit admitted through the fairness gate.
`0` or unset falls back to QPS-based rate limiting. | +| networking.kthenaRouter.fairness.outputTokenWeight | float | `2` | User-fairness strategy: weight multiplier for output tokens. | +| networking.kthenaRouter.fairness.windowSize | string | `"1h"` | User-fairness strategy: sliding window duration for token usage tracking. | +| networking.kthenaRouter.gatewayAPI.enabled | bool | `false` | Enable Gateway API related features. | +| networking.kthenaRouter.gatewayAPI.inferenceExtension | bool | `false` | Enable Gateway API Inference Extension features.
Requires `gatewayAPI.enabled` to be true. | +| networking.kthenaRouter.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for Kthena Router. | +| networking.kthenaRouter.image.repository | string | `"ghcr.io/volcano-sh/kthena-router"` | Image repository for Kthena Router. | +| networking.kthenaRouter.image.tag | string | `"latest"` | Image tag for Kthena Router. | +| networking.kthenaRouter.port | int | `8080` | Container port for Kthena Router. | +| networking.kthenaRouter.sessionBoost.enabled | bool | `false` | Enable session-boost scheduling. Mutually exclusive with fairness. | +| networking.kthenaRouter.sessionBoost.gracePeriod | string | `"0s"` | Wait time after a request completes for a same-session follow-up.
Disabled by default (`0s`). | +| networking.kthenaRouter.sessionBoost.header | string | `"X-Session-ID"` | HTTP header used to identify conversation sessions. | +| networking.kthenaRouter.sessionBoost.inflightPerPod | int | `16` | Maximum inflight requests admitted per backend pod.
Total inflight limit is this value times the number of backend pods.
`0` or unset uses the router default (16). | +| networking.kthenaRouter.sessionBoost.maxSessions | int | `4096` | Maximum number of recently-completed sessions kept warm for boosting.
Bounds an LRU cache; the least-recently-used session is evicted automatically.
Size it by the number of concurrent conversations to keep boosted. | +| networking.kthenaRouter.sessionBoost.timeout | string | `"30s"` | Maximum time a request may wait in the session-boost queue before it is
rejected with HTTP 504. Defaults to `30s`; set a non-positive duration
(e.g. `0s`) to disable it (bounded only by client disconnect). | +| networking.kthenaRouter.terminationGracePeriodSeconds | int | `330` | The router will drain all in-flight requests before forcefully closing connections. | +| networking.kthenaRouter.tls.dnsName | string | `"your-domain.com"` | DNS name to use for the certificate. | +| networking.kthenaRouter.tls.enabled | bool | `false` | Enable TLS for Kthena Router server. | +| networking.kthenaRouter.tls.secretName | string | `"kthena-router-tls"` | Secret name to store the certificate and key. | +| networking.kthenaRouter.webhook.enabled | bool | `true` | Enable webhook for Kthena Router. | +| networking.kthenaRouter.webhook.port | int | `8443` | Container port for Kthena Router webhook. | +| networking.kthenaRouter.webhook.servicePort | int | `443` | Service port for Kthena Router webhook. | +| networking.kthenaRouter.webhook.tls.certFile | string | `"/etc/tls/tls.crt"` | Certificate file path for the webhook. | +| networking.kthenaRouter.webhook.tls.keyFile | string | `"/etc/tls/tls.key"` | Key file path for the webhook. | +| networking.kthenaRouter.webhook.tls.secretName | string | `"kthena-router-webhook-certs"` | Secret name for storing webhook certificates. | +| workload.controllerManager.debugPort | int | `0` | Debug server port for Controller Manager (set 0 to disable). | +| workload.controllerManager.downloaderImage.repository | string | `"ghcr.io/volcano-sh/downloader"` | Image repository for the Downloader. | +| workload.controllerManager.downloaderImage.tag | string | `"latest"` | Image tag for the Downloader. | +| workload.controllerManager.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for the Controller Manager. | +| workload.controllerManager.image.repository | string | `"ghcr.io/volcano-sh/kthena-controller-manager"` | Image repository for the Controller Manager. | +| workload.controllerManager.image.tag | string | `"latest"` | Image tag for the Controller Manager. | +| workload.controllerManager.runtimeImage.repository | string | `"ghcr.io/volcano-sh/runtime"` | Image repository for the Runtime. | +| workload.controllerManager.runtimeImage.tag | string | `"latest"` | Image tag for the Runtime. | +| workload.controllerManager.webhook.enabled | bool | `true` | Enable webhook for the Controller Manager. | +| workload.controllerManager.webhook.tls.certSecretName | string | `"kthena-controller-manager-webhook-certs"` | Secret name for storing webhook certificates. | +| workload.controllerManager.webhook.tls.serviceName | string | `"kthena-controller-manager-webhook"` | Service name for the webhook. | +| workload.enabled | bool | `true` | Enable the workload subchart. | ## Notes From cbe82c40fd92e3bb20076e2d936cec0057118356 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Thu, 9 Jul 2026 11:34:09 +0000 Subject: [PATCH 11/11] fix comments Signed-off-by: YaoZengzeng --- pkg/kthena-router/datastore/fairness_queue.go | 34 +++++++++---------- .../datastore/session_boost_queue_test.go | 30 ++++++---------- pkg/kthena-router/router/router.go | 9 ++--- 3 files changed, 30 insertions(+), 43 deletions(-) diff --git a/pkg/kthena-router/datastore/fairness_queue.go b/pkg/kthena-router/datastore/fairness_queue.go index 25a6a47cb..099887fc2 100644 --- a/pkg/kthena-router/datastore/fairness_queue.go +++ b/pkg/kthena-router/datastore/fairness_queue.go @@ -184,19 +184,23 @@ func (r *Request) commitAdmission(fn func()) bool { } // Abandon marks the request as given up by the waiting caller (queue timeout, -// wait-reject, or client cancellation). It returns true if the request had -// already been admitted, in which case the caller owns the inflight permit and -// MUST call Release to avoid leaking capacity. When it returns false, admission is -// guaranteed not to proceed (commitAdmission observes abandoned and skips), so no -// permit can leak. -func (r *Request) Abandon() bool { +// wait-reject, or client cancellation). If the request had already been admitted, +// the caller owned the inflight permit, so Abandon releases it to avoid leaking +// capacity. Otherwise it marks the request abandoned so admission is guaranteed +// not to proceed (commitAdmission observes abandoned and skips) and no permit can +// leak. +func (r *Request) Abandon() { r.admitMu.Lock() - defer r.admitMu.Unlock() - if r.admitted { - return true + if !r.admitted { + r.abandoned = true + r.admitMu.Unlock() + return } - r.abandoned = true - return false + r.admitMu.Unlock() + // Admission raced in first, so we own the inflight permit; release it here. + // Release is guaranteed non-nil once admitted is set (installed by fn before + // commitAdmission sets admitted). + r.Release() } // RequestPriorityQueue implements the heap.Interface @@ -540,14 +544,10 @@ func (pq *RequestPriorityQueue) runSemaphoreMode(ctx context.Context) { req.Release = func() { releaseOnce.Do(func() { <-pq.sem - if pq.metrics != nil { - pq.metrics.DecFairnessQueueInflight(req.ModelName) - } + pq.metricDecInflight(req.ModelName) }) } - if pq.metrics != nil { - pq.metrics.IncFairnessQueueInflight(req.ModelName) - } + pq.metricIncInflight(req.ModelName) }) if !admitted { // Caller abandoned before admission; return the just-acquired permit and diff --git a/pkg/kthena-router/datastore/session_boost_queue_test.go b/pkg/kthena-router/datastore/session_boost_queue_test.go index 70afc2353..30859b3e2 100644 --- a/pkg/kthena-router/datastore/session_boost_queue_test.go +++ b/pkg/kthena-router/datastore/session_boost_queue_test.go @@ -573,9 +573,7 @@ func TestSessionBoost_AbandonBeforeAdmissionBlocksAdmission(t *testing.T) { } // Caller gives up first. - if admitted := req.Abandon(); admitted { - t.Fatal("Abandon() should report not-yet-admitted before admission") - } + req.Abandon() // The dequeue loop then tries to admit; it must skip. if q.admitSessionBoost(req) { @@ -592,8 +590,8 @@ func TestSessionBoost_AbandonBeforeAdmissionBlocksAdmission(t *testing.T) { } // TestSessionBoost_AdmissionBeforeAbandonReleases verifies that when admission wins -// the race, the caller's Abandon() reports the admission and can release the permit, -// returning the inflight count to zero. +// the race, the caller's Abandon() releases the permit it owns, returning the +// inflight count to zero. func TestSessionBoost_AdmissionBeforeAbandonReleases(t *testing.T) { cfg := sessionBoostConfig() q := newSessionBoostQueue(cfg, nil) @@ -617,17 +615,11 @@ func TestSessionBoost_AdmissionBeforeAbandonReleases(t *testing.T) { t.Fatalf("inflight count should be 1 after admission, got %d", got) } - // Caller times out after admission raced in: Abandon() must report admitted so - // the caller releases the permit. - if admitted := req.Abandon(); !admitted { - t.Fatal("Abandon() should report admitted after admission") - } - if req.Release == nil { - t.Fatal("Release must be installed once Abandon() reports admitted") - } - req.Release() + // Caller times out after admission raced in: Abandon() must release the permit + // it owns. + req.Abandon() if got := q.GetInflightCount(); got != 0 { - t.Fatalf("inflight count should return to 0 after Release, got %d", got) + t.Fatalf("inflight count should return to 0 after Abandon, got %d", got) } } @@ -655,13 +647,11 @@ func TestSessionBoost_ConcurrentAdmitAbandonNoLeak(t *testing.T) { defer wg.Done() q.admitSessionBoost(req) }() - // Handler side: time out and abandon; release if admission raced in. This - // mirrors the router's queue-wait timeout handling. + // Handler side: time out and abandon; Abandon releases if admission raced in. + // This mirrors the router's queue-wait timeout handling. go func() { defer wg.Done() - if req.Abandon() { - req.Release() - } + req.Abandon() }() wg.Wait() diff --git a/pkg/kthena-router/router/router.go b/pkg/kthena-router/router/router.go index cea5326fa..1cc668e0e 100644 --- a/pkg/kthena-router/router/router.go +++ b/pkg/kthena-router/router/router.go @@ -1186,12 +1186,9 @@ func (r *Router) handleFairnessScheduling(c *gin.Context, modelRequest ModelRequ return nil case <-reqCtx.Done(): // Abandon() atomically coordinates with the dequeue loop: if admission raced - // in first it returns true and we own the inflight permit, so release it here; - // otherwise it marks the request abandoned so the loop skips admission and no - // permit can leak. - if queueReq.Abandon() { - queueReq.Release() - } + // in first it releases the inflight permit we own; otherwise it marks the + // request abandoned so the loop skips admission and no permit can leak. + queueReq.Abandon() if errors.Is(reqCtx.Err(), context.DeadlineExceeded) { // Exceeded the queue-wait timeout. In session-boost mode this is expected // load-shedding when SESSION_BOOST_TIMEOUT is set, and under sustained