Skip to content

Add support for image content blocks in agent messages #197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Glavin001 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add support for image content blocks in agent messages #197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
30abdba
feat(opencode-adapter): handle ACP image content blocks in session/up…
claude Feb 27, 2026
842aba3
feat(inspector): render image content blocks in chat UI
claude Feb 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions frontend/packages/inspector/src/App.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -996,8 +996,25 @@ export default function App() {

switch (update.sessionUpdate) {
case "agent_message_chunk": {
const content = update.content as { type?: string; text?: string } | undefined;
if (content?.type === "text" && content.text) {
const content = update.content as { type?: string; text?: string; uri?: string; mimeType?: string } | undefined;
if (content?.type === "image" && content.uri) {
// Flush any accumulated text before the image
flushAssistant(time);
// Convert file:// URIs to the /v1/fs/file API endpoint so the
// browser can fetch the image from the Sandbox Agent server.
const filePath = content.uri.startsWith("file://")
? content.uri.slice("file://".length)
: content.uri;
const displayUri = `/v1/fs/file?path=${encodeURIComponent(filePath)}`;
entries.push({
id: `image-${event.id}`,
eventId: event.id,
kind: "image",
time,
role: "assistant",
image: { uri: displayUri, mimeType: content.mimeType ?? "image/png" },
});
} else if (content?.type === "text" && content.text) {
if (!assistantAccumId) {
assistantAccumId = `assistant-${event.id}`;
assistantAccumText = "";
Expand Down
10 changes: 8 additions & 2 deletions frontend/packages/inspector/src/components/chat/ChatMessages.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,14 +246,20 @@ const ChatMessages = ({
return <ToolGroup key={`group-${idx}`} entries={group.entries} onEventClick={onEventClick} />;
}

// Regular message
// Regular message or image
const entry = group.entries[0];
const messageClass = getMessageClass(entry);

return (
<div key={entry.id} className={`message ${messageClass} no-avatar`}>
<div className="message-content">
{entry.text ? (
{entry.image ? (
<img
src={entry.image.uri}
alt="Agent image"
style={{ maxWidth: "100%", borderRadius: 6 }}
/>
) : entry.text ? (
<MarkdownText text={entry.text} />
) : (
<span className="thinking-indicator">
Expand Down
4 changes: 3 additions & 1 deletion frontend/packages/inspector/src/components/chat/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
export type TimelineEntry = {
id: string;
eventId?: string; // Links back to the original event for navigation
kind: "message" | "tool" | "meta" | "reasoning";
kind: "message" | "tool" | "meta" | "reasoning" | "image";
time: string;
// For messages:
role?: "user" | "assistant";
Expand All @@ -15,4 +15,6 @@ export type TimelineEntry = {
reasoning?: { text: string; visibility?: string };
// For meta:
meta?: { title: string; detail?: string; severity?: "info" | "error" };
// For images:
image?: { uri: string; mimeType: string };
};
140 changes: 109 additions & 31 deletions server/packages/opencode-adapter/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3968,41 +3968,119 @@ async fn translate_session_update(
}

match kind {
// ── Text / thought chunk ───────────────────────────────────────
// ── Text / thought / image chunk ─────────────────────────────
"agent_message_chunk" | "agent_thought_chunk" => {
// ContentChunk.content is a ContentBlock; for text it has { type: "text", text: "…" }
let chunk = update
.pointer("/content/text")
// ContentChunk.content is a ContentBlock discriminated by `type`.
let content_type = update
.pointer("/content/type")
.and_then(Value::as_str)
.unwrap_or("");
if chunk.is_empty() {
return;
}
.unwrap_or("text");

match content_type {
// ── Image content block ──────────────────────────────────
"image" => {
// ACP ImageContent: { type: "image", uri: "file:///…", mimeType: "image/png" }
let uri = update
.pointer("/content/uri")
.and_then(Value::as_str)
.unwrap_or("");
if uri.is_empty() {
return;
}
let mime = update
.pointer("/content/mimeType")
.and_then(Value::as_str)
.unwrap_or("image/png");

// Accumulate into a single part — reuse the same part ID so the
// UI updates in-place instead of creating a new line per chunk.
text_accum.push_str(chunk);
let part_id = text_part_id.get_or_insert_with(|| {
let id = format!("part_{message_id}_{part_counter}");
*part_counter += 1;
id
});
let part = json!({
"id": *part_id,
"sessionID": session_id,
"messageID": message_id,
"type": "text",
"text": *text_accum,
});
state.emit_event(json!({
"type":"message.part.updated",
"properties":{
"sessionID": session_id,
"messageID": message_id,
"part": part,
"delta": chunk
// Derive a filesystem path from the URI for the filename field.
let path = uri.strip_prefix("file://").unwrap_or(uri);

// Finalize any accumulated text part before emitting the image.
if let Some(tid) = text_part_id.take() {
let part = json!({
"id": tid,
"sessionID": session_id,
"messageID": message_id,
"type": "text",
"text": *text_accum,
});
let env = json!({
"jsonrpc":"2.0",
"method":"_sandboxagent/opencode/message",
"params":{"message":{"info":{"id": message_id},"parts":[part]}}
});
if let Err(err) = state.persist_event(session_id, "agent", &env).await {
warn!(?err, "failed to persist ACP text part before image");
}
text_accum.clear();
}

let part_id = format!("part_{message_id}_{part_counter}");
*part_counter += 1;
let part = json!({
"id": part_id,
"sessionID": session_id,
"messageID": message_id,
"type": "file",
"mime": mime,
"filename": path,
"url": uri,
});
let env = json!({
"jsonrpc":"2.0",
"method":"_sandboxagent/opencode/message",
"params":{"message":{"info":{"id": message_id},"parts":[part.clone()]}}
});
if let Err(err) = state.persist_event(session_id, "agent", &env).await {
warn!(?err, "failed to persist ACP image part");
}
state.emit_event(json!({
"type":"message.part.updated",
"properties":{
"sessionID": session_id,
"messageID": message_id,
"part": part
}
}));
}
}));

// ── Text content block (default) ────────────────────────
_ => {
// For text: { type: "text", text: "…" }
let chunk = update
.pointer("/content/text")
.and_then(Value::as_str)
.unwrap_or("");
if chunk.is_empty() {
return;
}

// Accumulate into a single part — reuse the same part ID so the
// UI updates in-place instead of creating a new line per chunk.
text_accum.push_str(chunk);
let part_id = text_part_id.get_or_insert_with(|| {
let id = format!("part_{message_id}_{part_counter}");
*part_counter += 1;
id
});
let part = json!({
"id": *part_id,
"sessionID": session_id,
"messageID": message_id,
"type": "text",
"text": *text_accum,
});
state.emit_event(json!({
"type":"message.part.updated",
"properties":{
"sessionID": session_id,
"messageID": message_id,
"part": part,
"delta": chunk
}
}));
}
}
}

// ── Tool call initiation ───────────────────────────────────────
Expand Down