fix(anthropic): configurable max_tokens to prevent extended-thinking truncation (#45)

.env.example

@@ -62,6 +62,11 @@ BENCHMARK_JUDGE_MODEL=
 # Max number of models benchmarked in parallel
 BENCHMARK_CONCURRENCY=3
 
+# Max output tokens for Anthropic (Messages API) chat/judge/routing requests.
+# Extended-thinking models count thinking tokens toward this budget, so keep it
+# generous — too low can truncate the response before any answer text is produced.
+ANTHROPIC_MAX_TOKENS=12800
+
 # ─── Observability (OpenTelemetry + OpenMetrics) ───────────────────────────
 OTEL_ENABLED=false
 OTEL_SERVICE_NAME=pipimink

.github/copilot-instructions.md

@@ -146,6 +146,7 @@ Azure AI Foundry: one `ProviderConfig` entry per deployment, with `"models": ["m
 | `BENCHMARK_JUDGE_PROVIDER` | (selection provider) |
 | `BENCHMARK_JUDGE_MODEL` | (selection model) |
 | `BENCHMARK_CONCURRENCY` | `3` |
+| `ANTHROPIC_MAX_TOKENS` | `12800` (max output tokens for Anthropic chat/judge/routing; extended-thinking tokens count toward it) |
 | `SELECTION_CACHE_ENABLED` | `true` |
 | `SELECTION_CACHE_TTL` | `2m` |
 | `SELECTION_CACHE_MAX_ENTRIES` | `1000` |

AGENTS.md

@@ -138,6 +138,7 @@ Azure AI Foundry: one `ProviderConfig` entry per deployment, with `"models": ["m
 | `BENCHMARK_JUDGE_PROVIDER` | (selection provider) |
 | `BENCHMARK_JUDGE_MODEL` | (selection model) |
 | `BENCHMARK_CONCURRENCY` | `3` |
+| `ANTHROPIC_MAX_TOKENS` | `12800` (max output tokens for Anthropic chat/judge/routing; extended-thinking tokens count toward it) |
 | `SELECTION_CACHE_ENABLED` | `true` |
 | `SELECTION_CACHE_TTL` | `2m` |
 | `SELECTION_CACHE_MAX_ENTRIES` | `1000` |

CHANGELOG.md

@@ -83,6 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Provider config changes now propagate to the running LLM client immediately. The client held a provider snapshot from startup; a mutex-guarded `SetProviders()` is now invoked from every provider mutation handler, so chat, routing, and benchmark use the current config instead of a stale map (previously tagging worked but chat/benchmark hit the wrong endpoint).
 - Anthropic response parsing now scans all content blocks: extended-thinking models (e.g. Claude on Azure Foundry) emit a `thinking` block before the `text` block, which previously caused "missing/empty content" failures during tagging, chat, and benchmarking.
 - Anthropic policy refusals (`stop_reason=refusal`) are now surfaced as a distinct, clear error instead of a misleading "empty content" message.
+- Anthropic extended-thinking models no longer fail benchmarks/chat with "missing text in Anthropic content block". `max_tokens` was hard-coded to 4096, and thinking tokens count toward that budget, so harder prompts were cut off (`stop_reason=max_tokens`) before producing any answer text. `max_tokens` is now configurable via `ANTHROPIC_MAX_TOKENS` (default 12800) across chat, judge, and routing calls, and a `max_tokens` truncation is reported as a distinct, actionable error. (#45)
 - `scripts/start-stack.sh` now rebuilds the app image (`up -d --build`) so code changes are actually deployed to the running container.
 - Analytics latency time series query using incorrect `date_trunc` unit strings (`"1 hour"` → `"hour"`)
 

README.md

@@ -113,6 +113,8 @@ Routing decisions are cached in memory using a hash of the normalized prompt and
 
 Azure AI Foundry is supported via per-model `model_configs` entries. See [SETUP.md](SETUP.md#microsoft-azure-ai-foundry) for details.
 
+> Extended-thinking Claude models count their reasoning tokens toward the output budget. If answers get cut off, raise `ANTHROPIC_MAX_TOKENS` (default `12800`).
+
 ## Exposed APIs
 
 PiPiMink is a drop-in proxy. Existing clients require no changes:

SETUP.md

@@ -311,6 +311,12 @@ Scoring methods:
 | `BENCHMARK_SCHEDULE_ENABLED` | `false` | Run benchmarks automatically on a schedule |
 | `BENCHMARK_SCHEDULE_INTERVAL` | `24h` | Interval between scheduled benchmark runs |
 
+### Providers
+
+| Variable | Default | Description |
+| --- | --- | --- |
+| `ANTHROPIC_MAX_TOKENS` | `12800` | Max output tokens for Anthropic (Messages API) chat, judge, and routing requests. Extended-thinking models count thinking tokens toward this budget, so keep it generous — too low truncates the response before any answer text is produced. |
+
 ## Observability
 
 - Prometheus metrics: `GET /metrics`

internal/benchmark/scorer.go

@@ -17,15 +17,21 @@ import (
 type Scorer struct {
 	judgeProvider config.ProviderConfig // resolved provider config (with per-model overrides applied)
 	judgeModel    string
+	maxTokens     int // max_tokens for Anthropic judge requests
 	httpClient    *http.Client
 }
 
 // NewScorer creates a scorer backed by the given judge model endpoint.
 // The provider config should already have per-model overrides applied via ForModel().
-func NewScorer(judgeProvider config.ProviderConfig, judgeModel string) *Scorer {
+// maxTokens sets the Anthropic max_tokens budget; values <= 0 fall back to 12800.
+func NewScorer(judgeProvider config.ProviderConfig, judgeModel string, maxTokens int) *Scorer {
+	if maxTokens <= 0 {
+		maxTokens = 12800
+	}
 	return &Scorer{
 		judgeProvider: judgeProvider,
 		judgeModel:    judgeModel,
+		maxTokens:     maxTokens,
 		httpClient:    &http.Client{Timeout: judgeProvider.Timeout},
 	}
 }
@@ -205,7 +211,7 @@ func (s *Scorer) buildOpenAIRequest(systemMsg, userMsg string) ([]byte, string,
 func (s *Scorer) buildAnthropicRequest(systemMsg, userMsg string) ([]byte, string, error) {
 	payload := map[string]interface{}{
 		"model":      s.judgeModel,
-		"max_tokens": 4096,
+		"max_tokens": s.maxTokens,
 		"system":     systemMsg,
 		"messages": []map[string]string{
 			{"role": "user", "content": userMsg},
@@ -285,6 +291,9 @@ func extractAnthropicContent(body []byte) (string, error) {
 			return block.Text, nil
 		}
 	}
+	if result.StopReason == "max_tokens" {
+		return "", fmt.Errorf("response truncated at max_tokens before any text was produced (extended thinking consumed the token budget) — increase ANTHROPIC_MAX_TOKENS")
+	}
 	return "", fmt.Errorf("missing text in Anthropic content block")
 }
 

internal/benchmark/suite.go

@@ -44,7 +44,7 @@ func NewSuite(db DB, cfg *config.Config, chatFn ChatFunc) *Suite {
 	return &Suite{
 		db:     db,
 		cfg:    cfg,
-		scorer: NewScorer(judgeProvider, judgeModel),
+		scorer: NewScorer(judgeProvider, judgeModel, cfg.AnthropicMaxTokens),
 		chatFn: chatFn,
 	}
 }

internal/config/config.go

@@ -131,6 +131,9 @@ type Config struct {
 	BenchmarkJudgeModel       string // model for LLM judge (defaults to MODEL_SELECTION_MODEL)
 	BenchmarkConcurrency      int    // max parallel model benchmark runs (default 3)
 
+	// Anthropic
+	AnthropicMaxTokens int // max_tokens for Anthropic Messages API chat requests (default 12800). Extended-thinking models count thinking tokens toward this budget, so keep it generous.
+
 	// Persistence
 	DatabaseURL                   string
 	DatabaseMaxConnections        int
@@ -207,6 +210,7 @@ func Load() (*Config, error) {
 		BenchmarkJudgeProvider:         getEnv("BENCHMARK_JUDGE_PROVIDER", ""),
 		BenchmarkJudgeModel:            getEnv("BENCHMARK_JUDGE_MODEL", ""),
 		BenchmarkConcurrency:           getEnvInt("BENCHMARK_CONCURRENCY", 3),
+		AnthropicMaxTokens:             getEnvInt("ANTHROPIC_MAX_TOKENS", 12800),
 		OTelEnabled:                    getEnvBool("OTEL_ENABLED", false),
 		OTelServiceName:                getEnv("OTEL_SERVICE_NAME", "pipimink"),
 		OTelExporterOTLPEndpoint:       getEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "localhost:4318"),

internal/llm/chat.go

@@ -169,9 +169,14 @@ func (c *Client) chatWithAnthropicModel(p config.ProviderConfig, model string, m
 		}
 	}
 
+	maxTokens := c.Config.AnthropicMaxTokens
+	if maxTokens <= 0 {
+		maxTokens = 12800
+	}
+
 	payload := map[string]interface{}{
 		"model":      model,
-		"max_tokens": 4096,
+		"max_tokens": maxTokens,
 		"messages":   anthropicMessages,
 	}
 	if len(systemParts) > 0 {

internal/llm/model_selection.go

@@ -180,9 +180,13 @@ func (c *Client) DecideModelBasedOnCapabilities(message string, availableModels
 
 	switch selProvider.Type {
 	case config.ProviderTypeAnthropic:
+		maxTokens := c.Config.AnthropicMaxTokens
+		if maxTokens <= 0 {
+			maxTokens = 12800
+		}
 		payload := map[string]interface{}{
 			"model":      selectionModel,
-			"max_tokens": 4096,
+			"max_tokens": maxTokens,
 			"system":     systemMessage,
 			"messages": []map[string]string{
 				{"role": "user", "content": message},

internal/llm/model_selection_additional_test.go

@@ -66,7 +66,7 @@ func TestDecideModelBasedOnCapabilitiesWithAnthropicProvider(t *testing.T) {
 		err := json.NewDecoder(r.Body).Decode(&payload)
 		assert.NoError(t, err)
 		assert.NotEmpty(t, payload["system"], "system message should be top-level field")
-		assert.Equal(t, float64(4096), payload["max_tokens"])
+		assert.Equal(t, float64(12800), payload["max_tokens"])
 		msgs := payload["messages"].([]interface{})
 		assert.Len(t, msgs, 1, "should have only user message, no system message in messages array")
 		firstMsg := msgs[0].(map[string]interface{})

internal/llm/model_tags.go

@@ -501,6 +501,11 @@ func extractAnthropicContent(body []byte) (string, error) {
 			return text, nil
 		}
 	}
+	// A max_tokens stop with no text block means extended thinking consumed the
+	// entire output budget before producing an answer. Surface it distinctly.
+	if stop, _ := result["stop_reason"].(string); stop == "max_tokens" {
+		return "", fmt.Errorf("response truncated at max_tokens before any text was produced (extended thinking consumed the token budget) — increase ANTHROPIC_MAX_TOKENS")
+	}
 	return "", fmt.Errorf("missing text in Anthropic content block")
 }
 

internal/llm/model_tags_test.go

@@ -137,6 +137,14 @@ func TestExtractAnthropicContent(t *testing.T) {
 		assert.Contains(t, err.Error(), "model refused the request")
 		assert.Contains(t, err.Error(), "cyber content")
 	})
+
+	t.Run("Max tokens truncation with only thinking block", func(t *testing.T) {
+		body := []byte(`{"content":[{"type":"thinking","thinking":"lots of reasoning"}],"stop_reason":"max_tokens"}`)
+		_, err := extractAnthropicContent(body)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "max_tokens")
+		assert.Contains(t, err.Error(), "ANTHROPIC_MAX_TOKENS")
+	})
 }
 
 func TestDisableModelsWithEmptyTags(t *testing.T) {