Skip to content

Support google parameters #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
knjiang wants to merge 1 commit into
base: 01-28-add_google_test_cases
Choose a base branch
from
Open

Support google parameters #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/coverage-report/src/compact.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ pub struct PatternGroup {

/// Truncate a string to a maximum number of characters, adding "..." if truncated.
/// Uses character count, not byte count, to avoid UTF-8 panics.
fn truncate_str(s: &str, max_chars: usize) -> String {
pub fn truncate_str(s: &str, max_chars: usize) -> String {
let char_count = s.chars().count();
if char_count > max_chars {
let truncated: String = s.chars().take(max_chars.saturating_sub(3)).collect();
Expand Down
6 changes: 3 additions & 3 deletions crates/coverage-report/src/report.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ fn generate_markdown_report(
report.push_str(&format!(
" - `{}` - {}{}\n",
test_case,
error,
compact::truncate_str(&error, 200),
format_diff(&diff)
));
}
Expand Down Expand Up @@ -532,7 +532,7 @@ fn generate_markdown_report(
report.push_str(&format!(
" - `{}` - {}{}\n",
test_case,
error,
compact::truncate_str(&error, 200),
format_diff(&diff)
));
}
Expand Down Expand Up @@ -582,7 +582,7 @@ fn generate_markdown_report(
report.push_str(&format!(
" - `{}` - {}{}\n",
test_case,
error,
compact::truncate_str(&error, 200),
format_diff(&diff)
));
}
Expand Down
157 changes: 154 additions & 3 deletions crates/coverage-report/src/requests_expected_differences.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@
],
"errors": [
{ "pattern": "is not supported by OpenAI Chat Completions", "reason": "Provider-specific built-in tool has no OpenAI equivalent" },
{ "pattern": "Unsupported input type: UserContentPart variant: File", "reason": "Anthropic document blocks not supported in OpenAI" }
{ "pattern": "UserContentPart::File", "reason": "OpenAI doesn't support File content parts" },
{ "pattern": "AssistantContentPart::File", "reason": "OpenAI doesn't support File content in assistant messages" }
]
},
{
Expand All @@ -59,7 +60,8 @@
],
"errors": [
{ "pattern": "is not supported by OpenAI Responses API", "reason": "Provider-specific built-in tool has no OpenAI equivalent" },
{ "pattern": "Unsupported input type: UserContentPart variant: File", "reason": "Anthropic document blocks not supported in OpenAI" },
{ "pattern": "UserContentPart::File", "reason": "OpenAI doesn't support File content parts" },
{ "pattern": "AssistantContentPart::File", "reason": "OpenAI doesn't support File content in assistant messages" },
{ "pattern": "ToolResult { tool_name: \"web_search\"", "reason": "Anthropic web_search encrypted results cannot be transformed to OpenAI" }
]
},
Expand Down Expand Up @@ -108,7 +110,62 @@
{ "pattern": "params.stream", "reason": "Google uses endpoint-based streaming" },
{ "pattern": "params.metadata", "reason": "Google doesn't support metadata" },
{ "pattern": "params.reasoning.effort", "reason": "Cross-canonical conversion (effort→budget_tokens) may quantize when max_tokens is very small" },
{ "pattern": "params.reasoning.summary", "reason": "Google doesn't support reasoning summary" }
{ "pattern": "params.reasoning.summary", "reason": "Google doesn't support reasoning summary" },
{ "pattern": "params.parallel_tool_calls", "reason": "Google doesn't support parallel_tool_calls parameter" },
{ "pattern": "params.store", "reason": "Google doesn't support store parameter" },
{ "pattern": "params.frequency_penalty", "reason": "Google doesn't support frequency_penalty" },
{ "pattern": "params.presence_penalty", "reason": "Google doesn't support presence_penalty" },
{ "pattern": "params.seed", "reason": "Google doesn't support seed" },
{ "pattern": "params.logprobs", "reason": "Google doesn't support logprobs" },
{ "pattern": "params.top_logprobs", "reason": "Google doesn't support top_logprobs" },
{ "pattern": "params.tools[*].strict", "reason": "Google doesn't support strict mode for tools" },
{ "pattern": "params.tools[*].parameters.additionalProperties", "reason": "Google doesn't preserve additionalProperties in tool schemas" },
{ "pattern": "params.response_format.json_schema.schema.additionalProperties", "reason": "Google doesn't preserve additionalProperties in response schemas" },
{ "pattern": "params.response_format.json_schema.strict", "reason": "Google doesn't support strict mode for response schemas" },
{ "pattern": "params.tool_choice.disable_parallel", "reason": "Google doesn't support disable_parallel in tool_choice" },
{ "pattern": "params.temperature", "reason": "Google uses f32 for temperature, causing precision loss (0.7 → 0.699999988079071)" },
{ "pattern": "params.top_p", "reason": "Google uses f32 for top_p, causing precision loss (0.9 → 0.8999999761581421)" },
{ "pattern": "messages[*].content[*].media_type", "reason": "Google requires mime_type for images; null defaults to image/jpeg" },
{ "pattern": "messages[*].content[*].provider_options", "reason": "Google doesn't support provider_options on content parts" },
{ "pattern": "messages.length", "reason": "Google moves system messages to systemInstruction and merges consecutive same-role messages" }
],
"errors": [
{ "pattern": "is not supported by Google", "reason": "Provider-specific built-in tool has no Google equivalent" }
]
},
{
"source": "Responses",
"target": "Google",
"fields": [
{ "pattern": "params.tools", "reason": "OpenAI built-in tools (code_interpreter, web_search) not supported in Google" }
]
},
{
"source": "Anthropic",
"target": "Google",
"fields": [
{ "pattern": "params.tools", "reason": "Anthropic built-in tools (bash, text_editor, web_search) not supported in Google" }
]
},
{
"source": "Google",
"target": "*",
"fields": [
{ "pattern": "messages[*].content[*].encrypted_content", "reason": "Google thoughtSignature doesn't roundtrip through other providers" }
]
},
{
"source": "Google",
"target": "ChatCompletions",
"fields": [
{ "pattern": "messages[*].content[*].tool_name", "reason": "ChatCompletions tool messages don't include tool_name, only tool_call_id" }
]
},
{
"source": "Google",
"target": "Anthropic",
"fields": [
{ "pattern": "messages[*].content[*].tool_name", "reason": "Anthropic tool_result doesn't include tool name, only tool_use_id" }
]
}
],
Expand Down Expand Up @@ -252,6 +309,100 @@
"target": "ChatCompletions",
"skip": true,
"reason": "Anthropic document blocks not supported in OpenAI ChatCompletions"
},
{
"testCase": "reasoningEffortLowParam",
"source": "Google",
"target": "Responses",
"fields": [
{ "pattern": "params.reasoning.budget_tokens", "reason": "OpenAI uses effort levels, thinkingBudget gets quantized" }
]
},
{
"testCase": "reasoningEffortLowParam",
"source": "Google",
"target": "ChatCompletions",
"fields": [
{ "pattern": "params.reasoning.budget_tokens", "reason": "OpenAI uses effort levels, thinkingBudget gets quantized" }
]
},
{
"testCase": "reasoningSummaryParam",
"source": "Google",
"target": "Responses",
"fields": [
{ "pattern": "params.reasoning.budget_tokens", "reason": "OpenAI uses effort levels, thinkingBudget gets quantized" }
]
},
{
"testCase": "reasoningSummaryParam",
"source": "Google",
"target": "ChatCompletions",
"fields": [
{ "pattern": "params.reasoning.budget_tokens", "reason": "OpenAI uses effort levels, thinkingBudget gets quantized" }
]
},
{
"testCase": "thinkingLevelParam",
"source": "Google",
"target": "ChatCompletions",
"fields": [
{ "pattern": "messages[*].content.length", "reason": "ChatCompletions doesn't support thinking/reasoning content blocks" }
]
},
{
"testCase": "imageConfigParam",
"source": "Google",
"target": "ChatCompletions",
"fields": [
{ "pattern": "messages[*].content[*].type", "reason": "Google assistant image content not supported in ChatCompletions" },
{ "pattern": "messages[*].content[*].media_type", "reason": "Google assistant image content not supported in ChatCompletions" },
{ "pattern": "messages[*].content[*].data", "reason": "Google assistant image content not supported in ChatCompletions" },
{ "pattern": "messages[*].content[*].text", "reason": "Google assistant image content converted to text fallback in ChatCompletions" }
]
},
{
"testCase": "imageConfigParam",
"source": "Google",
"target": "Anthropic",
"fields": [
{ "pattern": "messages[*].content[*].type", "reason": "Google assistant image content not supported in Anthropic" },
{ "pattern": "messages[*].content[*].media_type", "reason": "Google assistant image content not supported in Anthropic" },
{ "pattern": "messages[*].content[*].data", "reason": "Google assistant image content not supported in Anthropic" },
{ "pattern": "messages[*].content[*].text", "reason": "Google assistant image content converted to text fallback in Anthropic" }
]
},
{
"testCase": "webSearchToolParam",
"source": "Anthropic",
"target": "Google",
"fields": [
{ "pattern": "messages[*].content.length", "reason": "Anthropic web search content blocks don't map 1:1 to Google structure" }
]
},
{
"testCase": "webSearchToolAdvancedParam",
"source": "Anthropic",
"target": "Google",
"fields": [
{ "pattern": "messages[*].content.length", "reason": "Anthropic web search content blocks don't map 1:1 to Google structure" }
]
},
{
"testCase": "documentContentParam",
"source": "Anthropic",
"target": "Google",
"fields": [
{ "pattern": "messages[*].content.length", "reason": "Anthropic document blocks not supported in Google" }
]
},
{
"testCase": "reasoningSummaryParam",
"source": "Google",
"target": "Responses",
"fields": [
{ "pattern": "messages.length", "reason": "Google reasoning summary creates additional messages in Responses format" }
]
}
]
}
109 changes: 109 additions & 0 deletions crates/coverage-report/src/responses_expected_differences.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,21 @@
{ "pattern": "params.service_tier", "reason": "OpenAI-specific billing tier not universal" }
]
},
{
"source": "Responses",
"target": "Google",
"fields": [
{ "pattern": "messages[*].content[*].provider_options", "reason": "Responses API annotations/logprobs have no Google equivalent" },
{ "pattern": "messages[*].content[*].provider_executed", "reason": "Responses API provider_executed flag has no Google equivalent" }
]
},
{
"source": "Google",
"target": "*",
"fields": [
{ "pattern": "messages[*].content[*].encrypted_content", "reason": "Google thoughtSignature doesn't roundtrip through other providers" }
]
},
{
"source": "Anthropic",
"target": "*",
Expand Down Expand Up @@ -152,6 +167,100 @@
"fields": [
{ "pattern": "messages.length", "reason": "Responses API doesn't support n>1 (multiple completions)" }
]
},
{
"testCase": "webSearchToolParam",
"source": "Anthropic",
"target": "Google",
"fields": [
{ "pattern": "messages[*].content.length", "reason": "Anthropic web search content blocks don't map 1:1 to Google structure" }
]
},
{
"testCase": "webSearchToolAdvancedParam",
"source": "Anthropic",
"target": "Google",
"fields": [
{ "pattern": "messages[*].content.length", "reason": "Anthropic web search content blocks don't map 1:1 to Google structure" }
]
},
{
"testCase": "thinkingLevelParam",
"source": "Google",
"target": "ChatCompletions",
"fields": [
{ "pattern": "messages[*].content.length", "reason": "ChatCompletions doesn't support thinking/reasoning content blocks" }
]
},
{
"testCase": "imageConfigParam",
"source": "Google",
"target": "ChatCompletions",
"fields": [
{ "pattern": "messages[*].content[*].type", "reason": "Google assistant image content not supported in ChatCompletions" },
{ "pattern": "messages[*].content[*].media_type", "reason": "Google assistant image content not supported in ChatCompletions" },
{ "pattern": "messages[*].content[*].data", "reason": "Google assistant image content not supported in ChatCompletions" },
{ "pattern": "messages[*].content[*].text", "reason": "Google assistant image content converted to text fallback in ChatCompletions" }
]
},
{
"testCase": "imageConfigParam",
"source": "Google",
"target": "Anthropic",
"fields": [
{ "pattern": "messages[*].content[*].type", "reason": "Google assistant image content not supported in Anthropic" },
{ "pattern": "messages[*].content[*].media_type", "reason": "Google assistant image content not supported in Anthropic" },
{ "pattern": "messages[*].content[*].data", "reason": "Google assistant image content not supported in Anthropic" },
{ "pattern": "messages[*].content[*].text", "reason": "Google assistant image content converted to text fallback in Anthropic" }
]
},
{
"testCase": "imageConfigParam",
"source": "Google",
"target": "Responses",
"fields": [
{ "pattern": "messages.length", "reason": "Google assistant image content not supported in Responses" }
]
},
{
"testCase": "webSearchToolParam",
"source": "Anthropic",
"target": "Google",
"fields": [
{ "pattern": "finish_reason", "reason": "Google infers finish_reason from content - presence of tool calls yields ToolCalls" }
]
},
{
"testCase": "webSearchToolAdvancedParam",
"source": "Anthropic",
"target": "Google",
"fields": [
{ "pattern": "finish_reason", "reason": "Google infers finish_reason from content - presence of tool calls yields ToolCalls" }
]
},
{
"testCase": "codeInterpreterToolParam",
"source": "Responses",
"target": "Google",
"fields": [
{ "pattern": "finish_reason", "reason": "Google infers finish_reason from content - presence of tool calls yields ToolCalls" }
]
},
{
"testCase": "webSearchToolParam",
"source": "Responses",
"target": "Google",
"fields": [
{ "pattern": "finish_reason", "reason": "Google infers finish_reason from content - presence of tool calls yields ToolCalls" }
]
},
{
"testCase": "reasoningSummaryParam",
"source": "Google",
"target": "Responses",
"fields": [
{ "pattern": "messages.length", "reason": "Google reasoning summary creates additional messages in Responses format" }
]
}
]
}
Loading