Skip to content

[MCP] Render MCP tool call result images to the model #5600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 27, 2025
Merged

[MCP] Render MCP tool call result images to the model #5600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2c53e50
Works
gpeal Oct 24, 2025
833bb1b
Cleanup + completions
gpeal Oct 26, 2025
604b797
Completions works
gpeal Oct 26, 2025
54c25d5
Cleanup
gpeal Oct 26, 2025
9a491fc
use From
gpeal Oct 27, 2025
5938f68
Merge remote-tracking branch 'origin/main' into gpeal/mcp-image
gpeal Oct 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 43 additions & 3 deletions codex-rs/core/src/chat_completions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use crate::util::backoff;
use bytes::Bytes;
use codex_otel::otel_event_manager::OtelEventManager;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ResponseItem;
use eventsource_stream::Eventsource;
Expand Down Expand Up @@ -159,16 +160,26 @@ pub(crate) async fn stream_chat_completions(
for (idx, item) in input.iter().enumerate() {
match item {
ResponseItem::Message { role, content, .. } => {
// Build content either as a plain string (typical for assistant text)
// or as an array of content items when images are present (user/tool multimodal).
let mut text = String::new();
let mut items: Vec<serde_json::Value> = Vec::new();
let mut saw_image = false;

for c in content {
match c {
ContentItem::InputText { text: t }
| ContentItem::OutputText { text: t } => {
text.push_str(t);
items.push(json!({"type":"text","text": t}));
}
ContentItem::InputImage { image_url } => {
saw_image = true;
items.push(json!({"type":"image_url","image_url": {"url": image_url}}));
}
_ => {}
}
}

// Skip exact-duplicate assistant messages.
if role == "assistant" {
if let Some(prev) = &last_assistant_text
Expand All @@ -179,7 +190,17 @@ pub(crate) async fn stream_chat_completions(
last_assistant_text = Some(text.clone());
}

let mut msg = json!({"role": role, "content": text});
// For assistant messages, always send a plain string for compatibility.
// For user messages, if an image is present, send an array of content items.
let content_value = if role == "assistant" {
json!(text)
} else if saw_image {
json!(items)
} else {
json!(text)
};

let mut msg = json!({"role": role, "content": content_value});
if role == "assistant"
&& let Some(reasoning) = reasoning_by_anchor_index.get(&idx)
&& let Some(obj) = msg.as_object_mut()
Expand Down Expand Up @@ -238,10 +259,29 @@ pub(crate) async fn stream_chat_completions(
messages.push(msg);
}
ResponseItem::FunctionCallOutput { call_id, output } => {
// Prefer structured content items when available (e.g., images)
// otherwise fall back to the legacy plain-string content.
let content_value = if let Some(items) = &output.content_items {
let mapped: Vec<serde_json::Value> = items
.iter()
.map(|it| match it {
FunctionCallOutputContentItem::InputText { text } => {
json!({"type":"text","text": text})
}
FunctionCallOutputContentItem::InputImage { image_url } => {
json!({"type":"image_url","image_url": {"url": image_url}})
}
})
.collect();
json!(mapped)
} else {
json!(output.content)
};

messages.push(json!({
"role": "tool",
"tool_call_id": call_id,
"content": output.content,
"content": content_value,
}));
}
ResponseItem::CustomToolCall {
Expand Down
51 changes: 10 additions & 41 deletions codex-rs/core/src/codex.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2047,7 +2047,7 @@ async fn try_run_turn(
call_id: String::new(),
output: FunctionCallOutputPayload {
content: msg.to_string(),
success: None,
..Default::default()
},
};
add_completed(ProcessedResponseItem {
Expand All @@ -2061,7 +2061,7 @@ async fn try_run_turn(
call_id: String::new(),
output: FunctionCallOutputPayload {
content: message,
success: None,
..Default::default()
},
};
add_completed(ProcessedResponseItem {
Expand Down Expand Up @@ -2199,41 +2199,6 @@ pub(super) fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -
}
})
}
pub(crate) fn convert_call_tool_result_to_function_call_output_payload(
call_tool_result: &CallToolResult,
) -> FunctionCallOutputPayload {
let CallToolResult {
content,
is_error,
structured_content,
} = call_tool_result;

// In terms of what to send back to the model, we prefer structured_content,
// if available, and fallback to content, otherwise.
let mut is_success = is_error != &Some(true);
let content = if let Some(structured_content) = structured_content
&& structured_content != &serde_json::Value::Null
&& let Ok(serialized_structured_content) = serde_json::to_string(&structured_content)
{
serialized_structured_content
} else {
match serde_json::to_string(&content) {
Ok(serialized_content) => serialized_content,
Err(err) => {
// If we could not serialize either content or structured_content to
// JSON, flag this as an error.
is_success = false;
err.to_string()
}
}
};

FunctionCallOutputPayload {
content,
success: Some(is_success),
}
}

/// Emits an ExitedReviewMode Event with optional ReviewOutput,
/// and records a developer message with the review output.
pub(crate) async fn exit_review_mode(
Expand Down Expand Up @@ -2443,14 +2408,15 @@ mod tests {
})),
};

let got = convert_call_tool_result_to_function_call_output_payload(&ctr);
let got = FunctionCallOutputPayload::from(&ctr);
let expected = FunctionCallOutputPayload {
content: serde_json::to_string(&json!({
"ok": true,
"value": 42
}))
.unwrap(),
success: Some(true),
..Default::default()
};

assert_eq!(expected, got);
Expand Down Expand Up @@ -2572,11 +2538,12 @@ mod tests {
structured_content: Some(serde_json::Value::Null),
};

let got = convert_call_tool_result_to_function_call_output_payload(&ctr);
let got = FunctionCallOutputPayload::from(&ctr);
let expected = FunctionCallOutputPayload {
content: serde_json::to_string(&vec![text_block("hello"), text_block("world")])
.unwrap(),
success: Some(true),
..Default::default()
};

assert_eq!(expected, got);
Expand All @@ -2590,10 +2557,11 @@ mod tests {
structured_content: Some(json!({ "message": "bad" })),
};

let got = convert_call_tool_result_to_function_call_output_payload(&ctr);
let got = FunctionCallOutputPayload::from(&ctr);
let expected = FunctionCallOutputPayload {
content: serde_json::to_string(&json!({ "message": "bad" })).unwrap(),
success: Some(false),
..Default::default()
};

assert_eq!(expected, got);
Expand All @@ -2607,10 +2575,11 @@ mod tests {
structured_content: None,
};

let got = convert_call_tool_result_to_function_call_output_payload(&ctr);
let got = FunctionCallOutputPayload::from(&ctr);
let expected = FunctionCallOutputPayload {
content: serde_json::to_string(&vec![text_block("alpha")]).unwrap(),
success: Some(true),
..Default::default()
};

assert_eq!(expected, got);
Expand Down
26 changes: 13 additions & 13 deletions codex-rs/core/src/conversation_history.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ impl ConversationHistory {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
..Default::default()
},
},
));
Expand Down Expand Up @@ -157,7 +157,7 @@ impl ConversationHistory {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
..Default::default()
},
},
));
Expand Down Expand Up @@ -454,7 +454,7 @@ mod tests {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
..Default::default()
},
},
];
Expand All @@ -470,7 +470,7 @@ mod tests {
call_id: "call-2".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
..Default::default()
},
},
ResponseItem::FunctionCall {
Expand Down Expand Up @@ -504,7 +504,7 @@ mod tests {
call_id: "call-3".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
..Default::default()
},
},
];
Expand Down Expand Up @@ -560,7 +560,7 @@ mod tests {
call_id: "call-x".to_string(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
..Default::default()
},
},
]
Expand Down Expand Up @@ -637,7 +637,7 @@ mod tests {
call_id: "shell-1".to_string(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
..Default::default()
},
},
]
Expand All @@ -651,7 +651,7 @@ mod tests {
call_id: "orphan-1".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
..Default::default()
},
}];
let mut h = create_history_with_items(items);
Expand Down Expand Up @@ -691,7 +691,7 @@ mod tests {
call_id: "c2".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
..Default::default()
},
},
// Will get an inserted custom tool output
Expand Down Expand Up @@ -733,7 +733,7 @@ mod tests {
call_id: "c1".to_string(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
..Default::default()
},
},
ResponseItem::CustomToolCall {
Expand Down Expand Up @@ -763,7 +763,7 @@ mod tests {
call_id: "s1".to_string(),
output: FunctionCallOutputPayload {
content: "aborted".to_string(),
success: None,
..Default::default()
},
},
]
Expand Down Expand Up @@ -828,7 +828,7 @@ mod tests {
call_id: "orphan-1".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
..Default::default()
},
}];
let mut h = create_history_with_items(items);
Expand Down Expand Up @@ -862,7 +862,7 @@ mod tests {
call_id: "c2".to_string(),
output: FunctionCallOutputPayload {
content: "ok".to_string(),
success: None,
..Default::default()
},
},
ResponseItem::CustomToolCall {
Expand Down
1 change: 1 addition & 0 deletions codex-rs/core/src/mcp_tool_call.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ pub(crate) async fn handle_mcp_tool_call(
output: FunctionCallOutputPayload {
content: format!("err: {e}"),
success: Some(false),
..Default::default()
},
};
}
Expand Down
7 changes: 2 additions & 5 deletions codex-rs/core/src/response_processing.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,11 @@ pub(crate) async fn process_items(
) => {
items_to_record_in_conversation_history.push(item);
let output = match result {
Ok(call_tool_result) => {
crate::codex::convert_call_tool_result_to_function_call_output_payload(
call_tool_result,
)
}
Ok(call_tool_result) => FunctionCallOutputPayload::from(call_tool_result),
Err(err) => FunctionCallOutputPayload {
content: err.clone(),
success: Some(false),
..Default::default()
},
};
items_to_record_in_conversation_history.push(ResponseItem::FunctionCallOutput {
Expand Down
Loading
Loading