diff --git a/sdkx/llm/anthropic/anthropic.go b/sdkx/llm/anthropic/anthropic.go index 442842c8..4ced3306 100644 --- a/sdkx/llm/anthropic/anthropic.go +++ b/sdkx/llm/anthropic/anthropic.go @@ -15,6 +15,7 @@ import ( asdk "github.com/anthropics/anthropic-sdk-go" sdkopt "github.com/anthropics/anthropic-sdk-go/option" "github.com/anthropics/anthropic-sdk-go/packages/param" + "github.com/anthropics/anthropic-sdk-go/shared/constant" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" @@ -89,6 +90,14 @@ func init() { // (read 2026-04-30) every current Claude SKU supports vision and // tool use, so only CapJSONSchema is disabled per family. // + // Audio input is not supported by the Anthropic Messages API at + // all — disable it so the caps middleware rejects audio parts up + // front instead of silently dropping them in convertContentParts. + // + // Frequency / presence penalty knobs do not exist on the Anthropic + // API either; disable them so callers don't think they're tuning + // sampling when the adapter is silently dropping the fields. + // // Output modality: text only. Claude has no native image or // audio output across the 4.x family — vision is input-only per // docs.anthropic.com/en/docs/vision (analyse / understand, @@ -96,6 +105,8 @@ func init() { // matching does not route image-output slots here. textOnlyOutput := llm.DisabledCaps( llm.CapJSONSchema, + llm.CapAudio, + llm.CapFrequencyPenalty, llm.CapPresencePenalty, llm.CapImageOutput, llm.CapAudioOutput, ) @@ -181,7 +192,11 @@ var _ llm.LLM = (*LLM)(nil) // anthropic.New call. Wrapping adapters override it via WithProviderName. const defaultProviderName = "anthropic" -// New creates an Anthropic LLM instance. +// New creates an Anthropic LLM instance. If httpClient is nil, a +// default client with a ResponseHeaderTimeout safety net is used so a +// hung Anthropic-compatible endpoint cannot block a caller indefinitely +// when no context deadline is set. The overall operation is still +// governed by the caller's context. func New(model, apiKey, baseURL string, httpClient *http.Client) (*LLM, error) { if model == "" { model = defaultModel @@ -194,13 +209,22 @@ func New(model, apiKey, baseURL string, httpClient *http.Client) (*LLM, error) { baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") ropts = append(ropts, sdkopt.WithBaseURL(baseURL)) } - if httpClient != nil { - ropts = append(ropts, sdkopt.WithHTTPClient(httpClient)) + if httpClient == nil { + httpClient = defaultHTTPClient() } + ropts = append(ropts, sdkopt.WithHTTPClient(httpClient)) client := asdk.NewClient(ropts...) return &LLM{client: client, model: asdk.Model(model), provider: defaultProviderName}, nil } +// defaultHTTPClient returns an http.Client with a ResponseHeaderTimeout +// safety net. This mirrors the timeout setup in sdkx/llm/bytedance. +func defaultHTTPClient() *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = 90 * time.Second + return &http.Client{Transport: transport} +} + // WithProviderName overrides the OTel / metrics provider tag used by // this LLM instance. Wrapping adapters (sdkx/llm/minimax) call this // so each sub-provider's calls land under its own name in traces and @@ -477,11 +501,6 @@ func convertMessages(messages []llm.Message) (system []asdk.TextBlockParam, out for _, msg := range messages { switch msg.Role { case llm.RoleSystem: - for _, p := range msg.Parts { - if p.Type != llm.PartText { - return nil, nil, errdefs.Validationf("anthropic: system message supports text parts only, got %s", p.Type) - } - } // One TextBlockParam per llm.Message{Role:System}: that // is the cache-segmentation primitive shared with // callers. Empty / whitespace-only segments are dropped @@ -548,12 +567,15 @@ func convertContentParts(parts []llm.Part) ([]asdk.ContentBlockParamUnion, error out = append(out, asdk.NewToolUseBlock(p.ToolCall.ID, json.RawMessage(p.ToolCall.Arguments), p.ToolCall.Name)) } case llm.PartImage: - if p.Image != nil && strings.HasPrefix(p.Image.URL, "data:") { - mediaType, b64, err := parseDataURL(p.Image.URL) - if err != nil { - return nil, err - } - out = append(out, asdk.NewImageBlockBase64(mediaType, b64)) + if p.Image == nil { + continue + } + blk, err := convertImagePartAnthropic(p.Image) + if err != nil { + return nil, err + } + if blk != nil { + out = append(out, *blk) } case llm.PartFile: if p.File != nil { @@ -567,27 +589,34 @@ func convertContentParts(parts []llm.Part) ([]asdk.ContentBlockParamUnion, error } case llm.PartData: if p.Data != nil { - text, err := formatAnthropicDataPartText(p.Data) - if err != nil { - return nil, err - } - out = append(out, asdk.NewTextBlock(text)) + b, _ := json.Marshal(p.Data.Value) + out = append(out, asdk.NewTextBlock(string(b))) } } } return out, nil } -func formatAnthropicDataPartText(data *llm.DataRef) (string, error) { - b, err := json.Marshal(data.Value) - if err != nil { - return "", errdefs.Validationf("anthropic: marshal data part: %v", err) - } - mime := strings.TrimSpace(data.MimeType) - if mime == "" { - mime = "application/json" +func convertImagePartAnthropic(img *llm.MediaRef) (*asdk.ContentBlockParamUnion, error) { + switch { + case img.URL == "" && img.Base64 == "": + return nil, nil + case strings.HasPrefix(img.URL, "data:"): + mediaType, b64, err := parseDataURL(img.URL) + if err != nil { + return nil, err + } + blk := asdk.NewImageBlockBase64(mediaType, b64) + return &blk, nil + case img.Base64 != "": + blk := asdk.NewImageBlockBase64(img.MediaType, img.Base64) + return &blk, nil + case strings.HasPrefix(img.URL, "http://") || strings.HasPrefix(img.URL, "https://"): + blk := asdk.NewImageBlock(asdk.URLImageSourceParam{URL: img.URL}) + return &blk, nil + default: + return nil, errdefs.Validationf("anthropic: unsupported image reference %q (expected data: URL, http(s) URL, or base64)", img.URL) } - return fmt.Sprintf("Claude input data\nMIME type: %s\nJSON:\n%s", mime, string(b)), nil } func convertFilePartAnthropic(f *llm.FileRef) (*asdk.ContentBlockParamUnion, error) { @@ -605,11 +634,21 @@ func convertFilePartAnthropic(f *llm.FileRef) (*asdk.ContentBlockParamUnion, err blk := asdk.NewDocumentBlock(asdk.URLPDFSourceParam{URL: f.URI}) return &blk, nil } - blk := asdk.NewDocumentBlock(asdk.PlainTextSourceParam{Data: f.URI}) - return &blk, nil + if strings.HasPrefix(f.URI, "data:") { + _, b64, err := parseDataURL(f.URI) + if err != nil { + return nil, err + } + blk := asdk.NewDocumentBlock(asdk.Base64PDFSourceParam{ + Data: b64, + MediaType: constant.ApplicationPDF("application/pdf"), + Type: constant.Base64("base64"), + }) + return &blk, nil + } + return nil, errdefs.Validationf("anthropic: PDF source must be an http(s) URL or a data: URI, got %q", f.URI) } - blk := asdk.NewDocumentBlock(asdk.PlainTextSourceParam{Data: f.URI}) - return &blk, nil + return nil, errdefs.Validationf("anthropic: unsupported file type %q (only application/pdf and image/* data: URIs are supported)", mime) } func convertResponse(blocks []asdk.ContentBlockUnion) llm.Message { diff --git a/sdkx/llm/anthropic/anthropic_test.go b/sdkx/llm/anthropic/anthropic_test.go index 4699f9d2..e062e474 100644 --- a/sdkx/llm/anthropic/anthropic_test.go +++ b/sdkx/llm/anthropic/anthropic_test.go @@ -4,10 +4,8 @@ import ( "context" "encoding/json" "io" - "math" "net/http" "net/http/httptest" - "strings" "testing" "github.com/GizClaw/flowcraft/sdk/errdefs" @@ -86,145 +84,15 @@ func TestGenerate_ThinkingTrueRejectsTooSmallMaxTokens(t *testing.T) { if err != nil { t.Fatalf("New: %v", err) } + maxTokens := int64(512) _, _, err = c.Generate(context.Background(), []llm.Message{ llm.NewTextMessage(llm.RoleUser, "hi"), - }, llm.WithMaxTokens(defaultThinkingBudgetTokens), llm.WithThinking(true)) - if !errdefs.IsValidation(err) { - t.Fatalf("expected validation error, got %v", err) - } - select { - case body := <-captured: - t.Fatalf("request should not have reached server, got body %#v", body) - default: - } -} - -func thinkingCaptureServer(t *testing.T, captured chan<- map[string]any) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - var body map[string]any - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { - t.Errorf("decode request body: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - select { - case captured <- body: - default: - t.Errorf("unexpected additional request body: %#v", body) - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{ - "id": "msg_test", - "type": "message", - "role": "assistant", - "model": "claude-3-sonnet-20240229", - "content": [{"type": "text", "text": "ok"}], - "stop_reason": "end_turn", - "usage": {"input_tokens": 1, "output_tokens": 1} - }`) - })) -} - -func readCapturedBody(t *testing.T, captured <-chan map[string]any) map[string]any { - t.Helper() - select { - case body := <-captured: - return body - default: - t.Fatal("server did not capture request body") - return nil - } -} - -func assertThinking(t *testing.T, body map[string]any, wantType string, wantBudget int64) { - t.Helper() - raw, ok := body["thinking"] - if !ok { - t.Fatalf("request body missing thinking: %#v", body) - } - thinking, ok := raw.(map[string]any) - if !ok { - t.Fatalf("thinking has unexpected shape: %#v", raw) - } - if got := thinking["type"]; got != wantType { - t.Fatalf("thinking.type = %v, want %q (thinking=%#v)", got, wantType, thinking) - } - if wantBudget == 0 { - if _, ok := thinking["budget_tokens"]; ok { - t.Fatalf("disabled thinking should not include budget_tokens: %#v", thinking) - } - return - } - gotBudget, ok := thinking["budget_tokens"].(float64) - if !ok { - t.Fatalf("thinking.budget_tokens missing or non-numeric: %#v", thinking) - } - if int64(gotBudget) != wantBudget { - t.Fatalf("thinking.budget_tokens = %v, want %d", gotBudget, wantBudget) - } -} - -// TestGenerate_NilResp_NoPanic regresses the same family of bug -// fixed in sdkx/llm/openai: anthropic-sdk-go's MessageService.New -// returns (*Message, error) and the pointer can be nil if the server -// answers with literal JSON null. Without the resp==nil guard, the -// next deref of resp.Content / resp.Usage would crash the goroutine. -// -// Triggered in production by MiniMax's /anthropic-compatible -// endpoint during degraded operation; the openai-go variant of the -// same bug crashed the LongMemEval _s eval at ~9% ingest. -func TestGenerate_NilResp_NoPanic(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = io.WriteString(w, "null") - })) - defer srv.Close() - - c, err := New("claude-3-sonnet-20240229", "test-key", srv.URL, nil) - if err != nil { - t.Fatalf("New: %v", err) - } - - defer func() { - if r := recover(); r != nil { - t.Fatalf("Generate panicked on nil resp: %v", r) - } - }() - - _, _, err = c.Generate(context.Background(), []llm.Message{llm.NewTextMessage(llm.RoleUser, "hi")}) + }, llm.WithThinking(true), llm.WithMaxTokens(maxTokens)) if err == nil { - t.Fatal("expected error, got nil") + t.Fatalf("expected error for thinking with max_tokens < budget, got none") } - if !errdefs.IsNotAvailable(err) { - t.Errorf("expected NotAvailable kind, got %v (%T)", err, err) - } - if !strings.Contains(err.Error(), "nil") { - t.Errorf("error message should mention nil, got %q", err.Error()) - } -} - -func TestGenerate_ThinkingFalseDisablesThinkingOnWire(t *testing.T) { - ts, cap := newCaptureServer(t) - c, err := New("claude-3-sonnet-20240229", "test-key", ts.URL, nil) - if err != nil { - t.Fatalf("New: %v", err) - } - - _, _, err = c.Generate( - context.Background(), - []llm.Message{llm.NewTextMessage(llm.RoleUser, "hi")}, - llm.WithThinking(false), - ) - if err != nil { - t.Fatalf("Generate: %v", err) - } - - rb := decodeBody(t, cap.body) - if rb.Thinking["type"] != "disabled" { - t.Fatalf("thinking = %#v, want type=disabled; body=%s", rb.Thinking, cap.body) + if !errdefs.IsValidation(err) { + t.Fatalf("expected Validation error, got %v", err) } } @@ -264,119 +132,138 @@ func TestGenerateStream_TransportError(t *testing.T) { _ = stream.Close() } -func TestConvertContentParts_DataPartUsesAnthropicTextBlock(t *testing.T) { - blocks, err := convertContentParts([]llm.Part{{ - Type: llm.PartData, - Data: &llm.DataRef{ - MimeType: "application/vnd.flowcraft.snapshot+json", - Value: map[string]any{"k": "v"}, - }, - }}) +// TestConvertImagePartAnthropic verifies that the image part converter +// accepts data: URLs, http(s) URLs, and raw base64 payloads. Previously +// only data: URLs were handled, so standard image URLs were silently dropped. +func TestConvertImagePartAnthropic(t *testing.T) { + // data: URL + dataURL := "data:image/png;base64,abcd" + blk, err := convertImagePartAnthropic(&llm.MediaRef{URL: dataURL}) if err != nil { - t.Fatalf("convertContentParts: %v", err) + t.Fatalf("data: URL: %v", err) } - if len(blocks) != 1 || blocks[0].OfText == nil { - t.Fatalf("expected one text block, got %#v", blocks) + if blk == nil || blk.OfImage == nil || blk.OfImage.Source.OfBase64 == nil { + t.Fatalf("data: URL did not produce base64 image block") } - text := blocks[0].OfText.Text - if !strings.Contains(text, "Claude input data") { - t.Fatalf("data block missing Claude label: %q", text) - } - if !strings.Contains(text, "MIME type: application/vnd.flowcraft.snapshot+json") { - t.Fatalf("data block missing mime_type: %q", text) + // https URL + blk, err = convertImagePartAnthropic(&llm.MediaRef{URL: "https://example.com/img.png"}) + if err != nil { + t.Fatalf("https URL: %v", err) } - if !strings.Contains(text, "JSON:\n{\"k\":\"v\"}") { - t.Fatalf("data block missing JSON content: %q", text) + if blk == nil || blk.OfImage == nil || blk.OfImage.Source.OfURL == nil || blk.OfImage.Source.OfURL.URL != "https://example.com/img.png" { + t.Fatalf("https URL did not produce URL image block") } -} -func TestConvertContentParts_DataPartDefaultsMimeType(t *testing.T) { - blocks, err := convertContentParts([]llm.Part{{ - Type: llm.PartData, - Data: &llm.DataRef{Value: map[string]any{"ok": true}}, - }}) + // raw base64 + media type + blk, err = convertImagePartAnthropic(&llm.MediaRef{Base64: "abcd", MediaType: "image/png"}) if err != nil { - t.Fatalf("convertContentParts: %v", err) + t.Fatalf("base64: %v", err) } - if len(blocks) != 1 || blocks[0].OfText == nil { - t.Fatalf("expected one text block, got %#v", blocks) + if blk == nil || blk.OfImage == nil || blk.OfImage.Source.OfBase64 == nil || blk.OfImage.Source.OfBase64.Data != "abcd" { + t.Fatalf("base64 did not produce base64 image block") } - if !strings.Contains(blocks[0].OfText.Text, "MIME type: application/json") { - t.Fatalf("empty mime_type should default to application/json: %q", blocks[0].OfText.Text) - } -} -func TestConvertContentParts_DataPartKeepsAdjacentTextBoundaries(t *testing.T) { - blocks, err := convertContentParts([]llm.Part{ - {Type: llm.PartText, Text: "before"}, - {Type: llm.PartData, Data: &llm.DataRef{Value: map[string]any{"n": float64(1)}}}, - {Type: llm.PartText, Text: "after"}, - }) + // empty + blk, err = convertImagePartAnthropic(&llm.MediaRef{}) if err != nil { - t.Fatalf("convertContentParts: %v", err) + t.Fatalf("empty: %v", err) } - if len(blocks) != 3 { - t.Fatalf("got %d blocks, want 3: %#v", len(blocks), blocks) + if blk != nil { + t.Fatalf("empty image ref should produce nil block") } - if blocks[0].OfText == nil || blocks[0].OfText.Text != "before" { - t.Fatalf("first text block changed: %#v", blocks[0]) + + // unsupported scheme + _, err = convertImagePartAnthropic(&llm.MediaRef{URL: "s3://bucket/img.png"}) + if err == nil || !errdefs.IsValidation(err) { + t.Fatalf("expected Validation error for unsupported scheme, got %v", err) } - if blocks[2].OfText == nil || blocks[2].OfText.Text != "after" { - t.Fatalf("last text block changed: %#v", blocks[2]) +} + +// TestConvertFilePartAnthropic verifies that PDF data: URIs are parsed +// into base64 document blocks and that non-PDF / non-image file types are +// rejected instead of having their URI sent as document text. +func TestConvertFilePartAnthropic(t *testing.T) { + // PDF data: URI -> base64 document block + pdfData := "data:application/pdf;base64,JVBERi0xLg==" + blk, err := convertFilePartAnthropic(&llm.FileRef{URI: pdfData, MimeType: "application/pdf"}) + if err != nil { + t.Fatalf("PDF data: URI: %v", err) } - if blocks[1].OfText == nil { - t.Fatalf("data block should be text, got %#v", blocks[1]) + if blk == nil || blk.OfDocument == nil || blk.OfDocument.Source.OfBase64 == nil || blk.OfDocument.Source.OfBase64.Data != "JVBERi0xLg==" { + t.Fatalf("PDF data: URI did not produce base64 document block") } - text := blocks[1].OfText.Text - if strings.Contains(text, "before") || strings.Contains(text, "after") { - t.Fatalf("data block should stay in its own Anthropic text block: %#v", blocks) + + // PDF https URL -> URL document block + blk, err = convertFilePartAnthropic(&llm.FileRef{URI: "https://example.com/doc.pdf", MimeType: "application/pdf"}) + if err != nil { + t.Fatalf("PDF https URL: %v", err) } - if !strings.Contains(text, "Claude input data") { - t.Fatalf("data block missing Claude label: %q", text) + if blk == nil || blk.OfDocument == nil || blk.OfDocument.Source.OfURL == nil || blk.OfDocument.Source.OfURL.URL != "https://example.com/doc.pdf" { + t.Fatalf("PDF https URL did not produce URL document block") } -} -func TestConvertMessages_SystemPartDataValidation(t *testing.T) { - _, _, err := convertMessages([]llm.Message{{ - Role: llm.RoleSystem, - Parts: []llm.Part{ - {Type: llm.PartText, Text: "rules"}, - {Type: llm.PartData, Data: &llm.DataRef{Value: map[string]any{"k": "v"}}}, - }, - }}) - if !errdefs.IsValidation(err) { - t.Fatalf("expected validation error, got %v", err) + // CSV URI must be rejected, not sent as plain text + _, err = convertFilePartAnthropic(&llm.FileRef{URI: "https://example.com/report.csv", MimeType: "text/csv"}) + if err == nil || !errdefs.IsValidation(err) { + t.Fatalf("expected Validation error for CSV, got %v", err) } - if !strings.Contains(err.Error(), "system message") { - t.Fatalf("error should mention system message, got %q", err.Error()) + + // Unsupported mime type + _, err = convertFilePartAnthropic(&llm.FileRef{URI: "data:text/plain;base64,abcd", MimeType: "text/plain"}) + if err == nil || !errdefs.IsValidation(err) { + t.Fatalf("expected Validation error for text/plain, got %v", err) } } -func TestGenerate_DataPartMarshalErrorIsValidation(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - t.Errorf("request should not be sent after data part validation fails") +func thinkingCaptureServer(t *testing.T, captured chan<- map[string]any) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("reading body: %v", err) + } + var parsed map[string]any + if err := json.Unmarshal(body, &parsed); err != nil { + t.Fatalf("unmarshal body: %v", err) + } + captured <- parsed + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{ + "id": "msg_01Test", + "type": "message", + "role": "assistant", + "content": [{"type":"text","text":"hi"}], + "model": "claude-3-sonnet-20240229", + "stop_reason": "end_turn", + "usage": {"input_tokens":5,"output_tokens":2} + }`) })) - defer srv.Close() +} - c, err := New("claude-3-sonnet-20240229", "test-key", srv.URL, nil) - if err != nil { - t.Fatalf("New: %v", err) +func readCapturedBody(t *testing.T, captured <-chan map[string]any) map[string]any { + t.Helper() + body := <-captured + if body == nil { + t.Fatalf("no captured body") } - msgs := []llm.Message{{ - Role: llm.RoleUser, - Parts: []llm.Part{{ - Type: llm.PartData, - Data: &llm.DataRef{Value: map[string]any{"bad": math.NaN()}}, - }}, - }} + return body +} - _, _, err = c.Generate(context.Background(), msgs) - if !errdefs.IsValidation(err) { - t.Fatalf("Generate error = %v, want Validation", err) +func assertThinking(t *testing.T, body map[string]any, wantType string, wantBudget int64) { + t.Helper() + thinking, ok := body["thinking"].(map[string]any) + if !ok { + t.Fatalf("expected thinking map, got %#v", body["thinking"]) } - _, err = c.GenerateStream(context.Background(), msgs) - if !errdefs.IsValidation(err) { - t.Fatalf("GenerateStream error = %v, want Validation", err) + if got := thinking["type"]; got != wantType { + t.Fatalf("thinking.type = %v, want %v", got, wantType) + } + if wantBudget > 0 { + if got := thinking["budget_tokens"]; got != float64(wantBudget) { + t.Fatalf("thinking.budget_tokens = %v, want %v", got, wantBudget) + } + } else if _, hasBudget := thinking["budget_tokens"]; hasBudget { + t.Fatalf("expected no budget_tokens for disabled thinking, got %#v", thinking) } } diff --git a/sdkx/llm/bytedance/bytedance.go b/sdkx/llm/bytedance/bytedance.go index 17a38f04..efd9dd7e 100644 --- a/sdkx/llm/bytedance/bytedance.go +++ b/sdkx/llm/bytedance/bytedance.go @@ -8,6 +8,31 @@ // routing locality is governed by Doubao's backend because ArkRuntime // does not expose a routing-hint field analogous to OpenAI's // `prompt_cache_key` or an explicit `cache_control` breakpoint. +// +// # Provider-specific Extra keys +// +// The Responses API surface is larger than the provider-agnostic +// [llm.GenerateOptions] exposes. The adapter honours these per-call +// keys via [llm.WithExtra]: +// +// - "previous_response_id" (string): reference a prior stored +// response to continue a conversation server-side. The referenced +// response must have been created with store=true (see below). +// - "store" (bool): whether the server stores this response for +// later retrieval via previous_response_id. Defaults to false — +// the adapter never reads responses back, so storing them only +// burns server-side quota. Set true only on the response you +// intend to continue. +// - "thinking" (string): "auto" | "enabled" | "disabled". Overrides +// [llm.WithThinking] and reaches the "auto" mode the bool option +// cannot express. When neither is set the field is omitted so the +// server applies its own default. +// - "reasoning_effort" (string): "minimal" | "low" | "medium" | +// "high". Maps to the Responses API `reasoning.effort` field; +// only meaningful when thinking is enabled/auto. +// - "web_search" (bool | map): enables the built-in web_search tool +// (also configurable at provider construction). +// - "parallel_tool_calls" (bool): forwarded to the Responses API. package bytedance import ( @@ -73,13 +98,25 @@ func init() { llm.CapStopWords, llm.CapFrequencyPenalty, llm.CapPresencePenalty, llm.CapAudio, ) + // Pro and Lite fix temperature=1 and top_p=0.95 server-side and + // silently ignore manual values (per the Doubao Seed 2.0 + // Responses API reference). Disable those caps on those two SKUs + // so the caps middleware fails fast instead of letting callers + // believe their sampling params took effect. Mini honours + // temperature/top_p, so it keeps the looser caps above. + chatTextOutputOnlyFixedSampling := llm.DisabledCaps( + llm.CapImageOutput, llm.CapAudioOutput, + llm.CapStopWords, llm.CapFrequencyPenalty, llm.CapPresencePenalty, + llm.CapAudio, + llm.CapTemperature, llm.CapTopP, + ) llm.RegisterProviderModels("bytedance", []llm.ModelInfo{ { Label: "Doubao Seed 2.0 Pro", Name: "doubao-seed-2-0-pro-260215", Spec: llm.ModelSpec{ - Caps: chatTextOutputOnly, + Caps: chatTextOutputOnlyFixedSampling, Limits: llm.ModelLimits{ MaxContextTokens: 256_000, MaxOutputTokens: 32_000, @@ -90,7 +127,7 @@ func init() { Label: "Doubao Seed 2.0 Lite", Name: "doubao-seed-2-0-lite-260215", Spec: llm.ModelSpec{ - Caps: chatTextOutputOnly, + Caps: chatTextOutputOnlyFixedSampling, Limits: llm.ModelLimits{ MaxContextTokens: 256_000, MaxOutputTokens: 32_000, @@ -146,7 +183,18 @@ func New(modelName, apiKey, baseURL, region string, retryTimes int) (*LLM, error if retryTimes > 0 { ropts = append(ropts, arkruntime.WithRetryTimes(retryTimes)) } - ropts = append(ropts, arkruntime.WithHTTPClient(http.DefaultClient)) + // http.DefaultClient has no Timeout and its DefaultTransport has + // no ResponseHeaderTimeout, so a server that accepts the TCP + // connection but never responds hangs forever when the caller + // forgot a context deadline. Clone the default transport (keeps + // its sane dial/TLS/idle timeouts) and add a response-header + // ceiling so a stuck connection fails fast. We deliberately do + // NOT set an overall Client.Timeout: legitimate streaming and + // long thinking runs can last minutes, and the context is the + // right cancellation channel for those. + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = 90 * time.Second + ropts = append(ropts, arkruntime.WithHTTPClient(&http.Client{Transport: transport})) client := arkruntime.NewClientWithApiKey(apiKey, ropts...) return &LLM{ @@ -182,6 +230,19 @@ func (c *LLM) Generate(ctx context.Context, messages []llm.Message, opts ...llm. } return llm.Message{}, llm.TokenUsage{}, classifyAPIError(err) } + // Same nil-(resp,err) defensive guard as the OpenAI adapter: the + // ark SDK's pointer-return convention is not a language guarantee, + // and a 200 with an undecodable body has been seen returning + // (nil, nil) on OpenAI-compatible backends. responseMessage is + // nil-safe on its own, but classifying explicitly keeps the error + // accurate and the two providers symmetric. + if resp == nil { + err := errdefs.NotAvailablef("bytedance: nil response with no error (provider misbehaviour)") + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + llm.RecordLLMMetrics(ctx, "bytedance", c.model, "error", dur, llm.TokenUsage{}) + return llm.Message{}, llm.TokenUsage{}, err + } msg := responseMessage(resp) if len(msg.Parts) == 0 { @@ -247,16 +308,10 @@ func (c *LLM) buildRequest(msgs []llm.Message, opts *llm.GenerateOptions) (*arkr } maxOutputTokens := int64(maxTokens) - thinkType := arkresponses.ThinkingType_disabled - if opts.Thinking != nil && *opts.Thinking { - thinkType = arkresponses.ThinkingType_enabled - } - req := &arkresponses.ResponsesRequest{ Model: c.model, Input: &arkresponses.ResponsesInput{Union: &arkresponses.ResponsesInput_ListValue{ListValue: &arkresponses.InputItemList{}}}, MaxOutputTokens: &maxOutputTokens, - Thinking: &arkresponses.ResponsesThinking{Type: &thinkType}, } if opts.Temperature != nil { @@ -327,6 +382,11 @@ func (c *LLM) buildRequest(msgs []llm.Message, opts *llm.GenerateOptions) (*arkr req.Tools = append(req.Tools, webSearch.tool()) } + // Responses API fields the provider-agnostic options cannot express + // (store, previous_response_id, thinking mode, reasoning effort). + // See the package doc for the supported Extra keys. + applyResponsesExtra(req, opts) + if err := appendResponsesMessages(req, msgs); err != nil { return nil, err } diff --git a/sdkx/llm/bytedance/responses.go b/sdkx/llm/bytedance/responses.go index 7e9c22a9..feed475b 100644 --- a/sdkx/llm/bytedance/responses.go +++ b/sdkx/llm/bytedance/responses.go @@ -253,16 +253,21 @@ func ensureResponsesTextBoundary(b *strings.Builder) { } } -func appendImageContent(items []*arkresponses.ContentItem, url string) []*arkresponses.ContentItem { - if url == "" { +func appendImageContent(items []*arkresponses.ContentItem, ref string) []*arkresponses.ContentItem { + if ref == "" { return items } - return append(items, &arkresponses.ContentItem{Union: &arkresponses.ContentItem_Image{ - Image: &arkresponses.ContentItemImage{ - Type: arkresponses.ContentItemType_input_image, - ImageUrl: &url, - }, - }}) + img := &arkresponses.ContentItemImage{Type: arkresponses.ContentItemType_input_image} + // The Responses API splits image-by-file-id from image-by-URL into + // separate fields; a file_id:// reference must not be stuffed into + // image_url or the server treats "file_id://..." as a (broken) URL. + if after, ok := strings.CutPrefix(ref, "file_id://"); ok { + img.FileId = stringPtrIfNotEmpty(after) + } else { + url := ref + img.ImageUrl = &url + } + return append(items, &arkresponses.ContentItem{Union: &arkresponses.ContentItem_Image{Image: img}}) } func appendFileContent(items []*arkresponses.ContentItem, fileRef *llm.FileRef) []*arkresponses.ContentItem { @@ -281,12 +286,35 @@ func appendFileContent(items []*arkresponses.ContentItem, fileRef *llm.FileRef) default: file.FileUrl = &fileRef.URI } - file.Filename = stringPtrIfNotEmpty(fileRef.Name) + if fileRef.Name != "" { + name := fileRef.Name + file.Filename = &name + } else if file.FileData != nil { + // Spec: filename is required when using file_data. Derive a + // usable name from the MIME type when the caller didn't give one. + if name := filenameFromMime(fileRef.MimeType); name != "" { + file.Filename = &name + } + } return append(items, &arkresponses.ContentItem{Union: &arkresponses.ContentItem_File{ File: file, }}) } +// filenameFromMime derives a "file." name from a MIME type for +// the file_data path, where the Responses API requires a filename. +// Returns "" for empty/unknown MIME so the caller leaves the field unset. +func filenameFromMime(mime string) string { + if mime == "" { + return "" + } + i := strings.IndexByte(mime, '/') + if i < 0 || i == len(mime)-1 { + return "" + } + return "file." + mime[i+1:] +} + func mediaURL(media *llm.MediaRef) string { if media.URL != "" { return media.URL @@ -343,6 +371,76 @@ func parseWebSearchConfig(v any) webSearchConfig { } } +// applyResponsesExtra maps Responses API fields the provider-agnostic +// [llm.GenerateOptions] cannot express onto req. Honoured Extra keys +// are documented on the package. store defaults to false because the +// adapter never reads responses back via previous_response_id in the +// common case, so leaving the server default (true) only burns quota. +func applyResponsesExtra(req *arkresponses.ResponsesRequest, opts *llm.GenerateOptions) { + store := false + if v, ok := opts.Extra["store"].(bool); ok { + store = v + } + req.Store = &store + + if v, ok := opts.Extra["previous_response_id"].(string); ok && v != "" { + req.PreviousResponseId = &v + } + + if thinkType, ok := parseThinkingExtra(opts.Extra["thinking"], opts.Thinking); ok { + t := thinkType + req.Thinking = &arkresponses.ResponsesThinking{Type: &t} + } + if effort, ok := parseReasoningEffortExtra(opts.Extra["reasoning_effort"]); ok { + req.Reasoning = &arkresponses.ResponsesReasoning{Effort: effort} + } +} + +// parseThinkingExtra resolves the thinking mode, giving Extra +// ("auto"/"enabled"/"disabled") precedence over the bool +// [llm.WithThinking] so callers can reach the "auto" mode the bool +// cannot express. Returns ok=false when neither is set, so the field +// is omitted and the server applies its own default. +func parseThinkingExtra(v any, opt *bool) (arkresponses.ThinkingType_Enum, bool) { + if s, ok := v.(string); ok { + switch strings.ToLower(s) { + case "auto": + return arkresponses.ThinkingType_auto, true + case "enabled": + return arkresponses.ThinkingType_enabled, true + case "disabled": + return arkresponses.ThinkingType_disabled, true + } + } + if opt != nil { + if *opt { + return arkresponses.ThinkingType_enabled, true + } + return arkresponses.ThinkingType_disabled, true + } + return 0, false +} + +// parseReasoningEffortExtra maps an effort string to the Responses +// API reasoning.effort enum. Unknown values are ignored (field omitted). +func parseReasoningEffortExtra(v any) (arkresponses.ReasoningEffort_Enum, bool) { + s, ok := v.(string) + if !ok { + return 0, false + } + switch strings.ToLower(s) { + case "minimal": + return arkresponses.ReasoningEffort_minimal, true + case "low": + return arkresponses.ReasoningEffort_low, true + case "medium": + return arkresponses.ReasoningEffort_medium, true + case "high": + return arkresponses.ReasoningEffort_high, true + } + return 0, false +} + func responseMessage(resp *arkresponses.ResponseObject) llm.Message { var parts []llm.Part for _, item := range resp.GetOutput() { @@ -546,12 +644,37 @@ func classifyResponseEventError(prefix, code, message string) error { msg += ": " + message } err := errdefs.Fmt("bytedance: %s", msg) - switch lower := strings.ToLower(code + " " + message); { + // Prefer an exact (case-insensitive) match on the provider's error + // code: providers reuse free-form messages that contain words like + // "context" or "auth" for unrelated errors, so substring matching on + // the message misclassifies. Fall back to substring only when the + // code is not a recognised token. + switch strings.ToLower(code) { + case "rate_limit", "rate_limit_exceeded", "ratelimit", "ratelimitexceeded", + "rate_limit_error", "429": + return errdefs.RateLimit(err) + case "unauthorized", "authentication_error", "auth_error", + "permission_error", "permission_denied", "401", "403": + return errdefs.Unauthorized(err) + case "forbidden", "insufficient_quota_error", "402": + return errdefs.Forbidden(err) + case "invalid_request_error", "invalid_parameter", "invalid", + "bad_request", "badrequest", "not_found", "not_found_error", + "validation_error", "400", "404": + return errdefs.Validation(err) + } + lower := strings.ToLower(code + " " + message) + switch { case strings.Contains(lower, "rate"): return errdefs.RateLimit(err) - case strings.Contains(lower, "auth") || strings.Contains(lower, "unauthorized") || strings.Contains(lower, "permission denied"): + case strings.Contains(lower, "permission denied"): + return errdefs.Unauthorized(err) + case strings.Contains(lower, "forbidden"): + return errdefs.Forbidden(err) + case strings.Contains(lower, "auth") || strings.Contains(lower, "unauthorized"): return errdefs.Unauthorized(err) - case strings.Contains(lower, "invalid") || strings.Contains(lower, "badrequest") || strings.Contains(lower, "notfound") || strings.Contains(lower, "context"): + case strings.Contains(lower, "invalid") || strings.Contains(lower, "badrequest") || + strings.Contains(lower, "notfound") || strings.Contains(lower, "context"): return errdefs.Validation(err) default: return errdefs.ClassifyProviderError("bytedance", err) diff --git a/sdkx/llm/bytedance/responses_extra_test.go b/sdkx/llm/bytedance/responses_extra_test.go new file mode 100644 index 00000000..e02a7335 --- /dev/null +++ b/sdkx/llm/bytedance/responses_extra_test.go @@ -0,0 +1,245 @@ +package bytedance + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/GizClaw/flowcraft/sdk/errdefs" + "github.com/GizClaw/flowcraft/sdk/llm" +) + +// newCaptureServer returns an httptest server that decodes the request +// body into got and replies with a minimal valid Responses payload. +func newCaptureServer(t *testing.T, got *map[string]any) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(got); err != nil { + t.Fatalf("decode request: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "output": []map[string]any{{ + "type": "message", + "role": "assistant", + "content": []map[string]any{{"type": "output_text", "text": "ok"}}, + }}, + }) + })) +} + +func TestBuildRequest_StoreDefaultsFalse(t *testing.T) { + var got map[string]any + srv := newCaptureServer(t, &got) + defer srv.Close() + + c, err := New("doubao-test", "test-key", srv.URL, "", 0) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, _, err := c.Generate(context.Background(), []llm.Message{llm.NewTextMessage(llm.RoleUser, "hi")}); err != nil { + t.Fatalf("Generate: %v", err) + } + if store, ok := got["store"].(bool); !ok || store { + t.Fatalf("store = %#v, want false (default)", got["store"]) + } +} + +func TestBuildRequest_StoreExtraOverride(t *testing.T) { + var got map[string]any + srv := newCaptureServer(t, &got) + defer srv.Close() + + c, err := New("doubao-test", "test-key", srv.URL, "", 0) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, _, err := c.Generate( + context.Background(), + []llm.Message{llm.NewTextMessage(llm.RoleUser, "hi")}, + llm.WithExtra("store", true), + ); err != nil { + t.Fatalf("Generate: %v", err) + } + if store, ok := got["store"].(bool); !ok || !store { + t.Fatalf("store = %#v, want true", got["store"]) + } +} + +func TestBuildRequest_PreviousResponseID(t *testing.T) { + var got map[string]any + srv := newCaptureServer(t, &got) + defer srv.Close() + + c, err := New("doubao-test", "test-key", srv.URL, "", 0) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, _, err := c.Generate( + context.Background(), + []llm.Message{llm.NewTextMessage(llm.RoleUser, "more")}, + llm.WithExtra("previous_response_id", "resp_123"), + ); err != nil { + t.Fatalf("Generate: %v", err) + } + if got["previous_response_id"] != "resp_123" { + t.Fatalf("previous_response_id = %#v", got["previous_response_id"]) + } +} + +func TestBuildRequest_ThinkingAutoViaExtra(t *testing.T) { + var got map[string]any + srv := newCaptureServer(t, &got) + defer srv.Close() + + c, err := New("doubao-test", "test-key", srv.URL, "", 0) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, _, err := c.Generate( + context.Background(), + []llm.Message{llm.NewTextMessage(llm.RoleUser, "prove Riemann")}, + llm.WithExtra("thinking", "auto"), + ); err != nil { + t.Fatalf("Generate: %v", err) + } + thinking, ok := got["thinking"].(map[string]any) + if !ok { + t.Fatalf("thinking = %#v, want object", got["thinking"]) + } + if thinking["type"] != "auto" { + t.Fatalf("thinking.type = %#v, want auto", thinking["type"]) + } +} + +func TestBuildRequest_ThinkingOmittedWhenNil(t *testing.T) { + var got map[string]any + srv := newCaptureServer(t, &got) + defer srv.Close() + + c, err := New("doubao-test", "test-key", srv.URL, "", 0) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, _, err := c.Generate( + context.Background(), + []llm.Message{llm.NewTextMessage(llm.RoleUser, "hi")}, + ); err != nil { + t.Fatalf("Generate: %v", err) + } + if _, present := got["thinking"]; present { + t.Fatalf("thinking = %#v, want omitted when neither opts.Thinking nor Extra set", got["thinking"]) + } +} + +func TestBuildRequest_ReasoningEffortViaExtra(t *testing.T) { + var got map[string]any + srv := newCaptureServer(t, &got) + defer srv.Close() + + c, err := New("doubao-test", "test-key", srv.URL, "", 0) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, _, err := c.Generate( + context.Background(), + []llm.Message{llm.NewTextMessage(llm.RoleUser, "think hard")}, + llm.WithExtra("thinking", "enabled"), + llm.WithExtra("reasoning_effort", "high"), + ); err != nil { + t.Fatalf("Generate: %v", err) + } + reasoning, ok := got["reasoning"].(map[string]any) + if !ok { + t.Fatalf("reasoning = %#v, want object", got["reasoning"]) + } + if reasoning["effort"] != "high" { + t.Fatalf("reasoning.effort = %#v, want high", reasoning["effort"]) + } +} + +func TestGenerate_UsesResponsesImageFileID(t *testing.T) { + var got map[string]any + srv := newCaptureServer(t, &got) + defer srv.Close() + + c, err := New("doubao-test", "test-key", srv.URL, "", 0) + if err != nil { + t.Fatalf("New: %v", err) + } + _, _, err = c.Generate(context.Background(), []llm.Message{{ + Role: llm.RoleUser, + Parts: []llm.Part{ + {Type: llm.PartFile, File: &llm.FileRef{URI: "file_id://img_123", MimeType: "image/png", Name: "pic.png"}}, + }, + }}) + if err != nil { + t.Fatalf("Generate: %v", err) + } + input := got["input"].([]any) + msg := input[0].(map[string]any) + content := msg["content"].([]any) + img := content[0].(map[string]any) + if img["type"] != "input_image" { + t.Fatalf("type = %#v", img["type"]) + } + if img["file_id"] != "img_123" { + t.Fatalf("file_id = %#v, want img_123", img["file_id"]) + } + if _, hasURL := img["image_url"]; hasURL { + t.Fatalf("image_url = %#v, want absent for file_id image", img["image_url"]) + } +} + +func TestGenerate_UsesResponsesFileDataFilenameDefault(t *testing.T) { + var got map[string]any + srv := newCaptureServer(t, &got) + defer srv.Close() + + c, err := New("doubao-test", "test-key", srv.URL, "", 0) + if err != nil { + t.Fatalf("New: %v", err) + } + _, _, err = c.Generate(context.Background(), []llm.Message{{ + Role: llm.RoleUser, + Parts: []llm.Part{ + {Type: llm.PartFile, File: &llm.FileRef{URI: "data:application/pdf;base64,JVBERi0=", MimeType: "application/pdf"}}, + }, + }}) + if err != nil { + t.Fatalf("Generate: %v", err) + } + input := got["input"].([]any) + msg := input[0].(map[string]any) + content := msg["content"].([]any) + file := content[0].(map[string]any) + if file["type"] != "input_file" { + t.Fatalf("type = %#v", file["type"]) + } + if file["filename"] != "file.pdf" { + t.Fatalf("filename = %#v, want file.pdf (derived from MIME)", file["filename"]) + } +} + +func TestClassifyResponseEventError_ExactCode(t *testing.T) { + tests := []struct { + code string + want func(error) bool + }{ + {"rate_limit_exceeded", errdefs.IsRateLimit}, + {"RateLimitExceeded", errdefs.IsRateLimit}, + {"authentication_error", errdefs.IsUnauthorized}, + {"forbidden", errdefs.IsForbidden}, + {"insufficient_quota_error", errdefs.IsForbidden}, + {"invalid_request_error", errdefs.IsValidation}, + {"not_found", errdefs.IsValidation}, + {"InvalidParameter", errdefs.IsValidation}, + } + for _, tc := range tests { + err := classifyResponseEventError("stream error", tc.code, "detail") + if !tc.want(err) { + t.Fatalf("code %q classified wrong: %v", tc.code, err) + } + } +} diff --git a/sdkx/llm/deepseek/deepseek.go b/sdkx/llm/deepseek/deepseek.go index f2ff3660..00786dcb 100644 --- a/sdkx/llm/deepseek/deepseek.go +++ b/sdkx/llm/deepseek/deepseek.go @@ -1,7 +1,6 @@ package deepseek import ( - "context" "time" "github.com/GizClaw/flowcraft/sdk/llm" @@ -54,11 +53,6 @@ func init() { llm.CapJSONSchema, llm.CapVision, llm.CapAudio, llm.CapFile, llm.CapImageOutput, llm.CapAudioOutput, ) - nonThinkingDefaults := llm.GenerateOptions{ - Extra: map[string]any{ - "thinking": map[string]any{"type": "disabled"}, - }, - } llm.RegisterProviderModels("deepseek", []llm.ModelInfo{ // --- V4 series (current) ------------------------------------- @@ -69,8 +63,7 @@ func init() { Label: "DeepSeek V4 Flash", Name: "deepseek-v4-flash", Spec: llm.ModelSpec{ - Caps: deepseekTextOnly, - Defaults: nonThinkingDefaults, + Caps: deepseekTextOnly, Limits: llm.ModelLimits{ MaxContextTokens: 1_000_000, MaxOutputTokens: 384_000, @@ -85,8 +78,7 @@ func init() { Label: "DeepSeek V4 Pro", Name: "deepseek-v4-pro", Spec: llm.ModelSpec{ - Caps: deepseekTextOnly, - Defaults: nonThinkingDefaults, + Caps: deepseekTextOnly, Limits: llm.ModelLimits{ MaxContextTokens: 1_000_000, MaxOutputTokens: 384_000, @@ -159,15 +151,18 @@ func init() { const ( defaultBaseURL = "https://api.deepseek.com/v1" - defaultModel = "deepseek-v4-pro" + // defaultModel uses the current non-deprecated SKU. The legacy + // deepseek-chat alias still resolves but is scheduled for hard + // retirement on 2026-07-24 (see legacyAliasRetiresAt and the + // catalog entries above). New zero-config deployments should not + // default to a model that disappears in two weeks. + defaultModel = "deepseek-v4-flash" ) -type LLM struct { - inner *openai.ChatLLM -} - -// New creates a DeepSeek LLM instance through the OpenAI-compatible Chat API. -func New(model, apiKey, baseURL string) (*LLM, error) { +// New creates a DeepSeek LLM instance (OpenAI-compatible). It uses the +// Chat Completions adapter rather than the default Responses adapter, +// because DeepSeek's API is OpenAI-compatible only on /chat/completions. +func New(model, apiKey, baseURL string) (*openai.ChatLLM, error) { if baseURL == "" { baseURL = defaultBaseURL } @@ -182,33 +177,5 @@ func New(model, apiKey, baseURL string) (*LLM, error) { // is observable as its own bucket instead of getting silently // aggregated under "openai" (the upstream HTTP transport). See // sdkx/llm/openai/openai.go ▸ WithProviderName. - inner.WithProviderName("deepseek") - return &LLM{inner: inner}, nil -} - -func (q *LLM) Generate(ctx context.Context, msgs []llm.Message, opts ...llm.GenerateOption) (llm.Message, llm.TokenUsage, error) { - return q.inner.Generate(ctx, msgs, injectChatThinking(opts)...) -} - -func (q *LLM) GenerateStream(ctx context.Context, msgs []llm.Message, opts ...llm.GenerateOption) (llm.StreamMessage, error) { - return q.inner.GenerateStream(ctx, msgs, injectChatThinking(opts)...) -} - -func (q *LLM) Provider() string { - if q == nil || q.inner == nil { - return "deepseek" - } - return q.inner.Provider() -} - -func injectChatThinking(opts []llm.GenerateOption) []llm.GenerateOption { - o := llm.ApplyOptions(opts...) - if o.Thinking == nil { - return opts - } - typ := "disabled" - if *o.Thinking { - typ = "enabled" - } - return append(opts, llm.WithExtra("thinking", map[string]any{"type": typ})) + return inner.WithProviderName("deepseek"), nil } diff --git a/sdkx/llm/deepseek/provider_test.go b/sdkx/llm/deepseek/provider_test.go index c3b7124f..8a9a550b 100644 --- a/sdkx/llm/deepseek/provider_test.go +++ b/sdkx/llm/deepseek/provider_test.go @@ -7,7 +7,6 @@ import ( "net/http" "net/http/httptest" "testing" - "time" "github.com/GizClaw/flowcraft/sdk/llm" ) @@ -16,7 +15,7 @@ import ( // must call openai.LLM.WithProviderName so traces/metrics from DeepSeek // traffic land under "deepseek" rather than the upstream "openai" tag. func TestNewTagsProvider(t *testing.T) { - c, err := New("deepseek-v4-flash", "k", "") + c, err := New("deepseek-chat", "k", "") if err != nil { t.Fatalf("New: %v", err) } @@ -25,112 +24,49 @@ func TestNewTagsProvider(t *testing.T) { } } -func TestProviderNilReceiverSafe(t *testing.T) { - var c *LLM - if got := c.Provider(); got != "deepseek" { - t.Fatalf("nil receiver Provider() = %q, want deepseek", got) - } -} - -func TestCatalogDefaultsDisableThinking(t *testing.T) { - for _, model := range []string{"deepseek-v4-flash", "deepseek-v4-pro"} { - spec := llm.DefaultRegistry.LookupModelSpec("deepseek", model) - thinking, ok := spec.Defaults.Extra["thinking"].(map[string]any) - if !ok { - t.Fatalf("%s thinking default = %#v, want object", model, spec.Defaults.Extra["thinking"]) +// TestNewDefaultsToCurrentSKU checks that the zero-config default model +// is the current non-deprecated SKU, not the legacy deepseek-chat alias +// that is scheduled for hard retirement on 2026-07-24. +func TestNewDefaultsToCurrentSKU(t *testing.T) { + var gotModel string + var sawAuth bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if h := r.Header.Get("Authorization"); h != "" { + sawAuth = true } - if got := thinking["type"]; got != "disabled" { - t.Fatalf("%s thinking.type = %#v, want disabled", model, got) + body, _ := io.ReadAll(r.Body) + var req struct { + Model string `json:"model"` } - } -} + _ = json.Unmarshal(body, &req) + gotModel = req.Model + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id":"test", + "object":"chat.completion", + "created":1, + "model":"` + req.Model + `", + "choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2} + }`)) + })) + defer srv.Close() -func TestDeepSeekResponsesProviderNotRegistered(t *testing.T) { - if _, err := llm.NewFromConfig("deepseek-responses", "deepseek-v4-flash", nil); err == nil { - t.Fatal("deepseek-responses should not be registered") + c, err := New("", "k", srv.URL) + if err != nil { + t.Fatalf("New: %v", err) } -} - -func TestGenerate_WithThinkingMapsToDeepSeekBody(t *testing.T) { - for _, tc := range []struct { - name string - opt llm.GenerateOption - want string - }{ - {name: "enabled", opt: llm.WithThinking(true), want: "enabled"}, - {name: "disabled", opt: llm.WithThinking(false), want: "disabled"}, - } { - t.Run(tc.name, func(t *testing.T) { - c, captured := newDeepSeekCaptureClient(t) - _, _, err := c.Generate(context.Background(), []llm.Message{ - llm.NewTextMessage(llm.RoleUser, "hi"), - }, tc.opt) - if err != nil { - t.Fatalf("Generate: %v", err) - } - body := readDeepSeekCapturedBody(t, captured) - thinking, ok := body["thinking"].(map[string]any) - if !ok { - t.Fatalf("thinking body field = %#v, want object; body=%#v", body["thinking"], body) - } - if got := thinking["type"]; got != tc.want { - t.Fatalf("thinking.type = %#v, want %q", got, tc.want) - } - }) + if got := c.Provider(); got != "deepseek" { + t.Fatalf("Provider() = %q, want deepseek", got) } -} - -func TestGenerate_CatalogDefaultDisablesDeepSeekThinkingOnWire(t *testing.T) { - c, captured := newDeepSeekCaptureClient(t) - spec := llm.DefaultRegistry.LookupModelSpec("deepseek", "deepseek-v4-flash") - wrapped := llm.WithDefaults(c, spec.Defaults) - _, _, err := wrapped.Generate(context.Background(), []llm.Message{ - llm.NewTextMessage(llm.RoleUser, "hi"), - }) + _, _, err = c.Generate(context.Background(), []llm.Message{llm.NewTextMessage(llm.RoleUser, "hello")}) if err != nil { t.Fatalf("Generate: %v", err) } - body := readDeepSeekCapturedBody(t, captured) - thinking, ok := body["thinking"].(map[string]any) - if !ok { - t.Fatalf("thinking body field = %#v, want object; body=%#v", body["thinking"], body) + if !sawAuth { + t.Fatalf("Authorization header missing") } - if got := thinking["type"]; got != "disabled" { - t.Fatalf("thinking.type = %#v, want disabled", got) - } -} - -func newDeepSeekCaptureClient(t *testing.T) (llm.LLM, <-chan map[string]any) { - t.Helper() - captured := make(chan map[string]any, 1) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/chat/completions" { - t.Fatalf("path = %q, want /chat/completions", r.URL.Path) - } - raw, _ := io.ReadAll(r.Body) - var body map[string]any - if err := json.Unmarshal(raw, &body); err != nil { - t.Fatalf("decode request body: %v\nbody=%s", err, raw) - } - captured <- body - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"id":"chatcmpl-test","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`) - })) - t.Cleanup(srv.Close) - c, err := New("deepseek-v4-flash", "k", srv.URL) - if err != nil { - t.Fatalf("New: %v", err) - } - return c, captured -} - -func readDeepSeekCapturedBody(t *testing.T, captured <-chan map[string]any) map[string]any { - t.Helper() - select { - case body := <-captured: - return body - case <-time.After(time.Second): - t.Fatal("server did not capture request body") + if gotModel != "deepseek-v4-flash" { + t.Fatalf("model on wire = %q, want deepseek-v4-flash", gotModel) } - return nil } diff --git a/sdkx/llm/minimax/anthropic.go b/sdkx/llm/minimax/anthropic.go index c09ed099..ca671d6c 100644 --- a/sdkx/llm/minimax/anthropic.go +++ b/sdkx/llm/minimax/anthropic.go @@ -12,8 +12,9 @@ func init() { return New(model, apiKey, baseURL) }) - // Catalog reflects MiniMax's M2-series public lineup as of - // 2026-04-30. Sources: + // Catalog reflects MiniMax's public lineup as of 2026-07-10. Sources: + // - https://platform.minimax.io/docs/api-reference/api-overview + // - https://platform.minimax.io/docs/api-reference/text-anthropic-api // - https://www.minimax.io/news/minimax-m25 // - https://github.com/MiniMax-AI/MiniMax-M2 // - https://developer.nvidia.com/blog/minimax-m2-7-advances-scalable-agentic-workflows-on-nvidia-platforms-for-complex-ai-applications/ @@ -21,8 +22,12 @@ func init() { // Notes: // - The M2 series is text-only at the model layer; image/video // features ship as a separate MiniMax MCP product, so - // CapVision / CapAudio / CapFile are disabled here to - // fail-fast at the caps middleware. + // CapVision / CapAudio / CapFile are disabled for M2.x entries. + // - MiniMax-M3 is the current flagship: 1M context, multimodal + // input (text, image, video), tool use, and thinking. It keeps + // CapVision enabled but still disables audio/file input (no + // public API support) and all output modalities (MiniMax-M3 is + // not an image/audio generator). // - This adapter dials MiniMax's Anthropic-compatible endpoint // (defaultBaseURL = ".../anthropic") through the Anthropic // Go SDK — see New() below, which delegates to @@ -40,25 +45,69 @@ func init() { // response_format on M2-series — see // github.com/MiniMax-AI/MiniMax-M2.5/issues/4. // response_format is only honored on legacy SKUs like - // MiniMax-Text-01 (not M2/M2.5/M2.7). + // MiniMax-Text-01 (not M2/M2.5/M2.7/M3). // - // Structured output on MiniMax M2-series today is - // prompt-engineering only (system-message instructions - // asking for JSON). The upstream caps middleware will pass - // such requests straight through. + // Structured output on MiniMax today is prompt-engineering only + // (system-message instructions asking for JSON). The upstream + // caps middleware will pass such requests straight through. // - // - Context window is 200K per the MiniMax-M2 model card. - textOnlyAnthropicStyle := llm.DisabledCaps( - llm.CapVision, llm.CapAudio, llm.CapFile, + // - The /anthropic endpoint docs list top_k, stop_sequences, + // frequency_penalty, and presence_penalty as ignored; disable + // the corresponding caps so callers get a clear signal instead + // of a silent no-op. The same docs list thinking as only + // accepting {"type":"adaptive"}, which the upstream Anthropic + // adapter emits as {"type":"enabled","budget_tokens":N}, so + // CapThinking is disabled for the M2.x entries. + // + // - Context window is 200K for M2.x and 1M for M3. + + // Cap set shared by M2.x and M3: parameters the /anthropic + // endpoint docs explicitly list as ignored, plus unsupported + // output modalities and JSON caps. + commonCaps := []llm.Capability{ + llm.CapAudio, llm.CapFile, llm.CapJSONMode, llm.CapJSONSchema, + llm.CapTopK, + llm.CapStopWords, + llm.CapFrequencyPenalty, llm.CapPresencePenalty, // Output: text only. Image generation lives in the // minimax-image adapter (sdkx/llm/minimax/image); audio // generation has no in-tree adapter today. llm.CapImageOutput, llm.CapAudioOutput, + } + + // M2.x is text-only and the MiniMax /anthropic docs list thinking + // as only {"type":"adaptive"}, while the upstream Anthropic SDK + // emits {"type":"enabled","budget_tokens":N}. Disable thinking and + // vision for this family. + textOnlyAnthropicStyle := llm.DisabledCaps( + append(append([]llm.Capability(nil), commonCaps...), llm.CapVision, llm.CapThinking)..., ) + // M3 keeps vision, tools, tool_choice, and thinking enabled; only + // audio/file input, JSON caps, and ignored sampling parameters are + // stripped. Output remains text-only. + multimodalAnthropicStyle := llm.DisabledCaps(commonCaps...) + llm.RegisterProviderModels("minimax", []llm.ModelInfo{ - // --- M2.7 generation (current default flagship) ------------- + // --- M3 generation (current flagship) ----------------------- + // Sources: + // - https://platform.minimax.io/docs/api-reference/api-overview + // - https://platform.minimax.io/docs/api-reference/text-anthropic-api + // + // 1M context, multimodal input (text/image/video), tool use, + // thinking, and interleaved thinking. Listed in the Anthropic- + // compatible endpoint, so this adapter can drive it directly. + { + Label: "MiniMax-M3", + Name: "MiniMax-M3", + Spec: llm.ModelSpec{ + Caps: multimodalAnthropicStyle, + Limits: llm.ModelLimits{MaxContextTokens: 1_000_000}, + }, + }, + + // --- M2.7 generation (previous flagship) -------------------- // Sources: // - https://platform.minimax.io/docs/api-reference/api-overview // - https://platform.minimax.io/docs/api-reference/text-anthropic-api @@ -66,10 +115,7 @@ func init() { // - https://blog.galaxy.ai/model/minimax-m2-7 // // Context window 204,800 tokens; max output ~131,072 tokens - // per Galaxy.ai snapshot. M2.7 is reasoning-only, text-only - // per artificialanalysis.ai. Listed in the Anthropic-compat - // API endpoint, so this adapter (which dials /anthropic) can - // drive it directly. + // per Galaxy.ai snapshot. M2.7 is reasoning-only, text-only. { Label: "MiniMax-M2.7", Name: "MiniMax-M2.7", @@ -122,6 +168,14 @@ func init() { Limits: llm.ModelLimits{MaxContextTokens: 204_800}, }, }, + { + Label: "MiniMax-M2.1 HighSpeed", + Name: "MiniMax-M2.1-highspeed", + Spec: llm.ModelSpec{ + Caps: textOnlyAnthropicStyle, + Limits: llm.ModelLimits{MaxContextTokens: 204_800}, + }, + }, // --- M2 (legacy base; not in Anthropic-compat list) -------- // Note: As of 2026-04-30 the docs page diff --git a/sdkx/llm/ollama/ollama.go b/sdkx/llm/ollama/ollama.go index 25d3ed68..e9f68bce 100644 --- a/sdkx/llm/ollama/ollama.go +++ b/sdkx/llm/ollama/ollama.go @@ -24,7 +24,17 @@ const defaultBaseURL = "http://127.0.0.1:11434" func init() { llm.RegisterProvider("ollama", func(model string, config map[string]any) (llm.LLM, error) { baseURL, _ := config["base_url"].(string) - return New(model, baseURL) + c, err := New(model, baseURL) + if err != nil { + return nil, err + } + // Ollama is a local-model runner: the actual capability set + // depends on whichever model the user has pulled. We cannot + // predeclare per-model caps, but we apply a provider-level + // safe default that disables features the Ollama /api/chat + // protocol itself does not support, so callers get a clear + // fail-fast instead of a silent drop. + return llm.WithCaps(c, defaultOllamaCaps), nil }) } @@ -37,6 +47,17 @@ type LLM struct { var _ llm.LLM = (*LLM)(nil) +// defaultOllamaCaps is the capability set applied to every Ollama model +// by New. Because Ollama models are user-defined and the registry does not +// know their names, we cannot look up per-model caps; instead we apply the +// provider-level safe default here. +var defaultOllamaCaps = llm.DisabledCaps( + llm.CapAudio, + llm.CapFile, + llm.CapJSONSchema, + llm.CapToolChoice, +) + // New creates an Ollama LLM instance. func New(model, baseURL string) (*LLM, error) { if baseURL == "" { @@ -47,10 +68,20 @@ func New(model, baseURL string) (*LLM, error) { return &LLM{ baseURL: baseURL, model: model, - httpClient: http.DefaultClient, + httpClient: defaultHTTPClient(), }, nil } +// defaultHTTPClient returns an http.Client with a ResponseHeaderTimeout +// safety net so a hung local Ollama server cannot block a caller +// indefinitely when no context deadline is set. The overall operation is +// still governed by the caller's context. +func defaultHTTPClient() *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = 90 * time.Second + return &http.Client{Transport: transport} +} + func (c *LLM) Generate(ctx context.Context, messages []llm.Message, opts ...llm.GenerateOption) (llm.Message, llm.TokenUsage, error) { ctx, span := telemetry.Tracer().Start(ctx, fmt.Sprintf("llm.ollama.generate.%s", c.model), trace.WithAttributes( attribute.String(telemetry.AttrLLMProvider, "ollama"), @@ -185,8 +216,13 @@ func (c *LLM) GenerateStream(ctx context.Context, messages []llm.Message, opts . } httpReq.Header.Set("Content-Type", "application/json") + start := time.Now() resp, err := c.httpClient.Do(httpReq) + dur := time.Since(start) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + llm.RecordLLMMetrics(ctx, "ollama", c.model, "error", dur, llm.TokenUsage{}) span.End() if ctx.Err() != nil { return nil, errdefs.FromContext(ctx.Err()) @@ -195,15 +231,23 @@ func (c *LLM) GenerateStream(ctx context.Context, messages []llm.Message, opts . } // See nil-resp guard rationale on the non-streaming path above. if resp == nil { + err := errdefs.NotAvailable(fmt.Errorf("ollama: nil http response with no error (custom transport misbehaviour)")) + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + llm.RecordLLMMetrics(ctx, "ollama", c.model, "error", dur, llm.TokenUsage{}) span.End() - return nil, errdefs.NotAvailable(fmt.Errorf("ollama: nil http response with no error (custom transport misbehaviour)")) + return nil, err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { defer func() { _ = resp.Body.Close() }() bodyBytes, _ := io.ReadAll(resp.Body) + statusErr := errdefs.ClassifyHTTPStatus("ollama", resp.StatusCode, strings.TrimSpace(string(bodyBytes))) + span.RecordError(statusErr) + span.SetStatus(codes.Error, statusErr.Error()) + llm.RecordLLMMetrics(ctx, "ollama", c.model, "error", dur, llm.TokenUsage{}) span.End() - return nil, errdefs.ClassifyHTTPStatus("ollama", resp.StatusCode, strings.TrimSpace(string(bodyBytes))) + return nil, statusErr } return newStreamMessage(ctx, span, c.model, resp.Body), nil diff --git a/sdkx/llm/ollama/stream.go b/sdkx/llm/ollama/stream.go index 8d8b8958..c4ecf398 100644 --- a/sdkx/llm/ollama/stream.go +++ b/sdkx/llm/ollama/stream.go @@ -7,7 +7,9 @@ import ( "io" "strings" "sync" + "time" + "github.com/GizClaw/flowcraft/sdk/errdefs" "github.com/GizClaw/flowcraft/sdk/llm" "github.com/GizClaw/flowcraft/sdk/telemetry" @@ -20,6 +22,7 @@ type ollamaStreamMessage struct { baseCtx context.Context span trace.Span model string + start time.Time mu sync.Mutex dec *json.Decoder @@ -46,6 +49,7 @@ func newStreamMessage( baseCtx: ctx, span: span, model: model, + start: time.Now(), dec: json.NewDecoder(body), body: body, } @@ -70,12 +74,12 @@ func (s *ollamaStreamMessage) Next() bool { if err := s.baseCtx.Err(); err != nil { body := s.body s.body = nil - s.err = err + s.err = errdefs.FromContext(err) s.mu.Unlock() if body != nil { _ = body.Close() } - s.finish(err) + s.finish(s.err) return false } dec := s.dec @@ -95,17 +99,17 @@ func (s *ollamaStreamMessage) Next() bool { s.finish(nil) return false } - s.mu.Lock() - body := s.body - s.body = nil - s.err = err - s.mu.Unlock() - if body != nil { - _ = body.Close() - } - s.finish(err) - return false + s.mu.Lock() + body := s.body + s.body = nil + s.err = errdefs.ClassifyProviderError("ollama", err) + s.mu.Unlock() + if body != nil { + _ = body.Close() } + s.finish(s.err) + return false + } s.updateFromChunk(chunk) s.accumulateToolCalls(chunk) @@ -227,9 +231,12 @@ func (s *ollamaStreamMessage) finish(err error) { } s.spanEnded = true usage := s.usage + dur := time.Since(s.start) s.mu.Unlock() + status := "success" if err != nil { + status = "error" s.span.RecordError(err) s.span.SetStatus(codes.Error, err.Error()) } else { @@ -239,5 +246,10 @@ func (s *ollamaStreamMessage) finish(err error) { ) s.span.SetStatus(codes.Ok, "OK") } + llm.RecordLLMMetrics(s.baseCtx, "ollama", s.model, status, dur, llm.TokenUsage{ + InputTokens: usage.InputTokens, + OutputTokens: usage.OutputTokens, + TotalTokens: usage.InputTokens + usage.OutputTokens, + }) s.span.End() } diff --git a/sdkx/llm/openai/chat/openai.go b/sdkx/llm/openai/chat/openai.go index e3ac06dd..74ebd336 100644 --- a/sdkx/llm/openai/chat/openai.go +++ b/sdkx/llm/openai/chat/openai.go @@ -141,11 +141,12 @@ func (c *LLM) Generate(ctx context.Context, messages []llm.Message, opts ...llm. TotalTokens: resp.Usage.TotalTokens, // usage.prompt_tokens_details.cached_tokens is the subset of // PromptTokens the provider served from its prefix cache. - // OpenAI / Azure OpenAI / DeepSeek / Qwen-flash populate it - // when the wire response carries prompt_tokens_details; - // older / lighter compatibles leave it zero, which is what - // we propagate via TokenUsage.CachedInputTokens (omitempty). - CachedInputTokens: resp.Usage.PromptTokensDetails.CachedTokens, + // OpenAI / Azure OpenAI / Qwen-flash populate it when the wire + // response carries prompt_tokens_details; older / lighter + // compatibles leave it zero. DeepSeek reports the same value as + // a top-level prompt_cache_hit_tokens, so we fall back to the + // raw response when the nested field is absent. + CachedInputTokens: openaishared.CachedInputTokensFromUsage(resp.Usage), } span.SetAttributes(llm.UsageSpanAttrs(usage)...) diff --git a/sdkx/llm/openai/chat/stream.go b/sdkx/llm/openai/chat/stream.go index 6d357cdb..5dd74b69 100644 --- a/sdkx/llm/openai/chat/stream.go +++ b/sdkx/llm/openai/chat/stream.go @@ -194,7 +194,7 @@ func (s *openaiStreamMessage) updateUsage(chunk oai.ChatCompletionChunk) { s.mu.Lock() s.usage.InputTokens = chunk.Usage.PromptTokens s.usage.OutputTokens = chunk.Usage.CompletionTokens - s.usage.CachedInputTokens = chunk.Usage.PromptTokensDetails.CachedTokens + s.usage.CachedInputTokens = openaishared.CachedInputTokensFromUsage(chunk.Usage) s.mu.Unlock() } diff --git a/sdkx/llm/openai/shared/cache.go b/sdkx/llm/openai/shared/cache.go index c668b07e..86e2fc2b 100644 --- a/sdkx/llm/openai/shared/cache.go +++ b/sdkx/llm/openai/shared/cache.go @@ -7,6 +7,8 @@ import ( "sort" "strings" + oai "github.com/openai/openai-go" + "github.com/GizClaw/flowcraft/sdk/llm" ) @@ -95,3 +97,25 @@ func writeCanonical(b *strings.Builder, v any) { b.Write(enc) } } + +// CachedInputTokensFromUsage extracts the cached-input token count from a +// chat-completion usage block. It prefers the standard OpenAI field +// usage.prompt_tokens_details.cached_tokens; if that is zero it falls back to +// the DeepSeek-compatible top-level field prompt_cache_hit_tokens, which some +// OpenAI-compatible providers (notably DeepSeek) emit instead. +func CachedInputTokensFromUsage(usage oai.CompletionUsage) int64 { + if cached := usage.PromptTokensDetails.CachedTokens; cached != 0 { + return cached + } + raw := usage.RawJSON() + if raw == "" { + return 0 + } + var body struct { + PromptCacheHitTokens int64 `json:"prompt_cache_hit_tokens"` + } + if err := json.Unmarshal([]byte(raw), &body); err != nil { + return 0 + } + return body.PromptCacheHitTokens +} diff --git a/sdkx/llm/openai/shared/cache_test.go b/sdkx/llm/openai/shared/cache_test.go index b346a259..d507c692 100644 --- a/sdkx/llm/openai/shared/cache_test.go +++ b/sdkx/llm/openai/shared/cache_test.go @@ -3,6 +3,8 @@ package shared import ( "testing" + oai "github.com/openai/openai-go" + "github.com/GizClaw/flowcraft/sdk/llm" ) @@ -116,6 +118,50 @@ func TestComputePromptCacheKey_NoStableContentEmpty(t *testing.T) { } } +// TestCachedInputTokensFromUsageFallback verifies that the DeepSeek-style +// top-level prompt_cache_hit_tokens field is used when the standard +// prompt_tokens_details.cached_tokens field is absent. +func TestCachedInputTokensFromUsageFallback(t *testing.T) { + cases := []struct { + name string + json string + want int64 + }{ + { + name: "openai nested cached_tokens", + json: `{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":80}}`, + want: 80, + }, + { + name: "deepseek top-level prompt_cache_hit_tokens", + json: `{"prompt_tokens":100,"prompt_cache_hit_tokens":70,"prompt_cache_miss_tokens":30,"completion_tokens":20,"total_tokens":120}`, + want: 70, + }, + { + name: "nested wins over top-level", + json: `{"prompt_tokens":100,"prompt_tokens_details":{"cached_tokens":80},"prompt_cache_hit_tokens":70}`, + want: 80, + }, + { + name: "no cache info", + json: `{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120}`, + want: 0, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + usage := oai.CompletionUsage{} + if err := usage.UnmarshalJSON([]byte(tc.json)); err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + got := CachedInputTokensFromUsage(usage) + if got != tc.want { + t.Fatalf("CachedInputTokensFromUsage() = %d, want %d", got, tc.want) + } + }) + } +} + // TestComputePromptCacheKey_SchemaKeyOrderStable: Go map iteration is // non-deterministic, but the cache key must be - canonicalJSON sorts // keys so two equivalent schemas hash identically. diff --git a/sdkx/llm/qwen/provider_test.go b/sdkx/llm/qwen/provider_test.go index fcb774fa..44899015 100644 --- a/sdkx/llm/qwen/provider_test.go +++ b/sdkx/llm/qwen/provider_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "testing" - "github.com/GizClaw/flowcraft/sdk/errdefs" "github.com/GizClaw/flowcraft/sdk/llm" ) @@ -27,112 +26,49 @@ func TestNewTagsProvider(t *testing.T) { } } -func TestNewUsesDashScopeResponsesBaseURL(t *testing.T) { - var got map[string]any +// TestQwenExtrasOnWire verifies that Qwen-specific options are mapped to +// the legacy/extra body fields the DashScope OpenAI-compatible endpoint +// actually honors: max_tokens (not max_completion_tokens), top_k, and +// enable_thinking. +func TestQwenExtrasOnWire(t *testing.T) { + var reqBody map[string]any srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v2/apps/protocols/compatible-mode/v1/responses" { - t.Fatalf("path = %q, want DashScope Responses path", r.URL.Path) - } - if err := json.NewDecoder(r.Body).Decode(&got); err != nil { - t.Fatalf("decode request: %v", err) - } + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &reqBody) w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"id":"resp_1","object":"response","created_at":0,"model":"qwen-flash","output":[{"type":"message","id":"msg_1","status":"completed","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}`) + _, _ = w.Write([]byte(`{ + "id":"test", + "object":"chat.completion", + "created":1, + "model":"qwen-flash", + "choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2} + }`)) })) defer srv.Close() - c, err := New("qwen-flash", "test-key", srv.URL+"/api/v2/apps/protocols/compatible-mode/v1") + maxTokens := int64(300) + topK := int64(50) + thinking := true + c, err := New("qwen-flash", "k", srv.URL) if err != nil { t.Fatalf("New: %v", err) } - msg, _, err := c.Generate(context.Background(), []llm.Message{llm.NewTextMessage(llm.RoleUser, "hi")}) + _, _, err = c.Generate(context.Background(), []llm.Message{llm.NewTextMessage(llm.RoleUser, "hello")}, + llm.WithMaxTokens(maxTokens), + llm.WithTopK(topK), + llm.WithThinking(thinking), + ) if err != nil { t.Fatalf("Generate: %v", err) } - if msg.Content() != "ok" || got["model"] != "qwen-flash" { - t.Fatalf("content=%q body=%#v", msg.Content(), got) + if got, ok := reqBody["max_tokens"]; !ok || got != float64(maxTokens) { + t.Fatalf("max_tokens on wire = %v (%v), want %v", got, ok, maxTokens) } -} - -func TestResponsesBaseURLNormalization(t *testing.T) { - tests := []struct { - name string - in string - want string - }{ - { - name: "empty uses China default", - in: "", - want: defaultResponsesBaseURL, - }, - { - name: "China chat base switches to responses base", - in: "https://dashscope.aliyuncs.com/compatible-mode/v1", - want: defaultResponsesBaseURL, - }, - { - name: "international chat base switches to responses base", - in: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", - want: "https://dashscope-intl.aliyuncs.com/api/v2/apps/protocols/compatible-mode/v1", - }, - { - name: "responses base stays unchanged", - in: "https://dashscope-intl.aliyuncs.com/api/v2/apps/protocols/compatible-mode/v1/", - want: "https://dashscope-intl.aliyuncs.com/api/v2/apps/protocols/compatible-mode/v1", - }, - { - name: "custom base stays explicit", - in: "https://example.test/custom/v1", - want: "https://example.test/custom/v1", - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := qwenResponsesBaseURL(tc.in); got != tc.want { - t.Fatalf("qwenResponsesBaseURL(%q) = %q, want %q", tc.in, got, tc.want) - } - }) + if got, ok := reqBody["top_k"]; !ok || got != float64(topK) { + t.Fatalf("top_k on wire = %v (%v), want %v", got, ok, topK) } -} - -func TestNewMapsThinkingToResponsesEnableThinking(t *testing.T) { - for _, tc := range []struct { - name string - opts []llm.GenerateOption - want bool - }{ - {name: "default disabled", want: false}, - {name: "enabled", opts: []llm.GenerateOption{llm.WithThinking(true)}, want: true}, - {name: "disabled", opts: []llm.GenerateOption{llm.WithThinking(false)}, want: false}, - } { - t.Run(tc.name, func(t *testing.T) { - var got map[string]any - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(&got); err != nil { - t.Fatalf("decode request: %v", err) - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"id":"resp_1","object":"response","created_at":0,"model":"qwen-flash","output":[{"type":"message","id":"msg_1","status":"completed","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}`) - })) - defer srv.Close() - - c, err := New("qwen-flash", "test-key", srv.URL) - if err != nil { - t.Fatalf("New: %v", err) - } - if _, _, err := c.Generate(context.Background(), []llm.Message{llm.NewTextMessage(llm.RoleUser, "hi")}, tc.opts...); err != nil { - t.Fatalf("Generate: %v", err) - } - if got["enable_thinking"] != tc.want { - t.Fatalf("enable_thinking = %#v, want %v; body=%#v", got["enable_thinking"], tc.want, got) - } - }) - } -} - -func TestQwenChatProviderIsNotRegistered(t *testing.T) { - _, err := llm.NewFromConfig("qwen-chat", "qwen-flash", map[string]any{"api_key": "test-key"}) - if !errdefs.IsNotFound(err) { - t.Fatalf("qwen-chat registration error = %v, want NotFound", err) + if got, ok := reqBody["enable_thinking"]; !ok || got != true { + t.Fatalf("enable_thinking on wire = %v (%v), want true", got, ok) } } diff --git a/sdkx/llm/qwen/qwen.go b/sdkx/llm/qwen/qwen.go index e6c44767..03d3d141 100644 --- a/sdkx/llm/qwen/qwen.go +++ b/sdkx/llm/qwen/qwen.go @@ -2,7 +2,6 @@ package qwen import ( "context" - "strings" "github.com/GizClaw/flowcraft/sdk/llm" "github.com/GizClaw/flowcraft/sdkx/llm/openai" @@ -42,10 +41,12 @@ func init() { // image-output / audio-output slots onto these chat models. qwenChatCaps := llm.DisabledCaps( llm.CapAudio, llm.CapFile, + llm.CapJSONSchema, + llm.CapFrequencyPenalty, llm.CapImageOutput, llm.CapAudioOutput, ) - qwenModels := []llm.ModelInfo{ + llm.RegisterProviderModels("qwen", []llm.ModelInfo{ { // Flagship; 256K context per Model Studio docs. Label: "Qwen Max", @@ -86,69 +87,64 @@ func init() { Limits: llm.ModelLimits{MaxContextTokens: 1_000_000}, }, }, - } - llm.RegisterProviderModels("qwen", qwenModels) + }) } const ( - defaultResponsesBaseURL = "https://dashscope.aliyuncs.com/api/v2/apps/protocols/compatible-mode/v1" - defaultModel = "qwen-flash" + defaultBaseURL = "https://dashscope.aliyuncs.com/compatible-mode/v1" + defaultModel = "qwen-flash" ) -// LLM wraps the OpenAI-compatible Responses adapter to handle Qwen-specific parameters. +// LLM wraps openai.ChatLLM to handle Qwen-specific parameters. type LLM struct { - inner *openai.LLM + inner *openai.ChatLLM } -// New creates a Qwen LLM instance backed by DashScope's Responses API. +// New creates a Qwen LLM instance. Wraps the Chat Completions adapter +// to inject enable_thinking based on GenerateOptions.Thinking. func New(model, apiKey, baseURL string) (*LLM, error) { + if baseURL == "" { + baseURL = defaultBaseURL + } if model == "" { model = defaultModel } - inner, err := openai.New(model, apiKey, qwenResponsesBaseURL(baseURL)) + inner, err := openai.NewChat(model, apiKey, baseURL) if err != nil { return nil, err } // Tag the OTel/metrics provider as "qwen" so dashboards split out - // Qwen traffic from the upstream OpenAI-compatible transport. See - // sdkx/llm/openai/openai.go ▸ WithProviderName for the contract. + // Qwen traffic from the upstream openai.ChatLLM that delegates the HTTP + // transport. See sdkx/llm/openai/openai.go ▸ WithProviderName for + // the contract. inner.WithProviderName("qwen") return &LLM{inner: inner}, nil } func (q *LLM) Generate(ctx context.Context, msgs []llm.Message, opts ...llm.GenerateOption) (llm.Message, llm.TokenUsage, error) { - return q.inner.Generate(ctx, msgs, injectResponsesThinking(opts)...) + return q.inner.Generate(ctx, msgs, append(opts, qwenExtras(opts)...)...) } func (q *LLM) GenerateStream(ctx context.Context, msgs []llm.Message, opts ...llm.GenerateOption) (llm.StreamMessage, error) { - return q.inner.GenerateStream(ctx, msgs, injectResponsesThinking(opts)...) -} - -func (q *LLM) Provider() string { - if q == nil || q.inner == nil { - return "qwen" - } - return q.inner.Provider() + return q.inner.GenerateStream(ctx, msgs, append(opts, qwenExtras(opts)...)...) } -func qwenResponsesBaseURL(baseURL string) string { - baseURL = strings.TrimRight(baseURL, "/") - if baseURL == "" { - return defaultResponsesBaseURL - } - if strings.HasSuffix(baseURL, "/api/v2/apps/protocols/compatible-mode/v1") { - return baseURL +// qwenExtras maps Qwen-specific GenerateOptions fields to Extra body keys +// that the OpenAI-compatible baseline does not emit. The baseline sends +// max_completion_tokens, which Qwen's compatible endpoint silently ignores; +// Qwen still honors the legacy max_tokens field, so we mirror MaxTokens +// there. top_k is a Qwen-specific sampling parameter the baseline never +// sets. enable_thinking is Qwen's thinking toggle. +func qwenExtras(opts []llm.GenerateOption) []llm.GenerateOption { + o := llm.ApplyOptions(opts...) + out := []llm.GenerateOption{ + llm.WithExtra("enable_thinking", o.Thinking != nil && *o.Thinking), } - if before, ok := strings.CutSuffix(baseURL, "/compatible-mode/v1"); ok { - return before + "/api/v2/apps/protocols/compatible-mode/v1" + if o.MaxTokens != nil { + out = append(out, llm.WithExtra("max_tokens", *o.MaxTokens)) } - return baseURL -} - -func injectResponsesThinking(opts []llm.GenerateOption) []llm.GenerateOption { - o := llm.ApplyOptions(opts...) - if o.Thinking == nil { - return append(opts, llm.WithExtra("enable_thinking", false)) + if o.TopK != nil { + out = append(out, llm.WithExtra("top_k", *o.TopK)) } - return append(opts, llm.WithExtra("enable_thinking", *o.Thinking)) + return out }