Skip to content

Add image support to ACP agent message handling #196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Glavin001 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add image support to ACP agent message handling #196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2a55147
feat(claude): add image support for Claude Code agent
claude Feb 27, 2026
7d3409a
fix(claude): correct error_events and file_attachments in capability …
claude Feb 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/session-transcript-schema.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ This table shows which agent feature coverage appears in the universal event str
| Tool Results | ✓ | ✓ | ✓ | ✓ | ✓ |
| Questions (HITL) | ✓ | | ✓ | | |
| Permissions (HITL) | ✓ | ✓ | ✓ | - | |
| Images | - | ✓ | ✓ | - | ✓ |
| File Attachments | - | ✓ | ✓ | - | |
| Images | ✓ | ✓ | ✓ | - | ✓ |
| File Attachments | | ✓ | ✓ | - | |
| Session Lifecycle | - | ✓ | ✓ | - | |
| Error Events | - | ✓ | ✓ | ✓ | ✓ |
| Error Events | ✓ | ✓ | ✓ | ✓ | ✓ |
| Reasoning/Thinking | - | ✓ | - | - | ✓ |
| Command Execution | - | ✓ | - | - | |
| File Changes | - | ✓ | - | - | |
Expand Down
158 changes: 127 additions & 31 deletions server/packages/opencode-adapter/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3968,41 +3968,126 @@ async fn translate_session_update(
}

match kind {
// ── Text / thought chunk ───────────────────────────────────────
// ── Text / thought / image chunk ──────────────────────────────
"agent_message_chunk" | "agent_thought_chunk" => {
// ContentChunk.content is a ContentBlock; for text it has { type: "text", text: "…" }
let chunk = update
.pointer("/content/text")
let content_type = update
.pointer("/content/type")
.and_then(Value::as_str)
.unwrap_or("");
if chunk.is_empty() {
return;
}
.unwrap_or("text");

// Accumulate into a single part — reuse the same part ID so the
// UI updates in-place instead of creating a new line per chunk.
text_accum.push_str(chunk);
let part_id = text_part_id.get_or_insert_with(|| {
let id = format!("part_{message_id}_{part_counter}");
*part_counter += 1;
id
});
let part = json!({
"id": *part_id,
"sessionID": session_id,
"messageID": message_id,
"type": "text",
"text": *text_accum,
});
state.emit_event(json!({
"type":"message.part.updated",
"properties":{
"sessionID": session_id,
"messageID": message_id,
"part": part,
"delta": chunk
match content_type {
"image" => {
// ACP image content: { type: "image", data: "<base64>", mimeType: "image/png", uri?: "..." }
let data = update
.pointer("/content/data")
.and_then(Value::as_str)
.unwrap_or("");
let mime_type = update
.pointer("/content/mimeType")
.and_then(Value::as_str)
.unwrap_or("image/png");
let uri = update
.pointer("/content/uri")
.and_then(Value::as_str);

if data.is_empty() && uri.is_none() {
return;
}

// Finalize any accumulated text part before the image.
if let Some(tid) = text_part_id.take() {
let part = json!({
"id": tid,
"sessionID": session_id,
"messageID": message_id,
"type": "text",
"text": *text_accum,
});
let env = json!({
"jsonrpc":"2.0",
"method":"_sandboxagent/opencode/message",
"params":{"message":{"info":{"id": message_id},"parts":[part]}}
});
if let Err(err) = state.persist_event(session_id, "agent", &env).await {
warn!(?err, "failed to persist ACP text part before image");
}
text_accum.clear();
}

let part_id = format!("part_{message_id}_{part_counter}");
*part_counter += 1;

// Build a file part with the image data. If we have a URI,
// use it directly; otherwise embed the base64 data as a data-URI.
let url = if let Some(u) = uri {
u.to_string()
} else {
format!("data:{};base64,{}", mime_type, data)
};

let part = json!({
"id": part_id,
"sessionID": session_id,
"messageID": message_id,
"type": "file",
"mime": mime_type,
"url": url,
"filename": format!("image.{}", mime_extension(mime_type)),
});

let env = json!({
"jsonrpc":"2.0",
"method":"_sandboxagent/opencode/message",
"params":{"message":{"info":{"id": message_id},"parts":[part.clone()]}}
});
if let Err(err) = state.persist_event(session_id, "agent", &env).await {
warn!(?err, "failed to persist ACP image part");
}
state.emit_event(json!({
"type":"message.part.updated",
"properties":{
"sessionID": session_id,
"messageID": message_id,
"part": part
}
}));
}
}));
_ => {
// ContentChunk.content is a ContentBlock; for text it has { type: "text", text: "…" }
let chunk = update
.pointer("/content/text")
.and_then(Value::as_str)
.unwrap_or("");
if chunk.is_empty() {
return;
}

// Accumulate into a single part — reuse the same part ID so the
// UI updates in-place instead of creating a new line per chunk.
text_accum.push_str(chunk);
let part_id = text_part_id.get_or_insert_with(|| {
let id = format!("part_{message_id}_{part_counter}");
*part_counter += 1;
id
});
let part = json!({
"id": *part_id,
"sessionID": session_id,
"messageID": message_id,
"type": "text",
"text": *text_accum,
});
state.emit_event(json!({
"type":"message.part.updated",
"properties":{
"sessionID": session_id,
"messageID": message_id,
"part": part,
"delta": chunk
}
}));
}
}
}

// ── Tool call initiation ───────────────────────────────────────
Expand Down Expand Up @@ -4122,6 +4207,17 @@ async fn translate_session_update(
}
}

fn mime_extension(mime: &str) -> &str {
match mime {
"image/png" => "png",
"image/jpeg" => "jpg",
"image/gif" => "gif",
"image/webp" => "webp",
"image/svg+xml" => "svg",
_ => "png",
}
}

fn normalize_proxy_base_url(value: String) -> Option<String> {
let trimmed = value.trim();
if trimmed.is_empty() {
Expand Down
4 changes: 2 additions & 2 deletions server/packages/sandbox-agent/src/router/support.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,10 +205,10 @@ pub(super) fn agent_capabilities_for(agent: AgentId) -> AgentCapabilities {
tool_calls: true,
tool_results: true,
text_messages: true,
images: false,
images: true,
file_attachments: false,
session_lifecycle: false,
error_events: false,
error_events: true,
reasoning: false,
status: false,
command_execution: false,
Expand Down