From 30abdba4209e868e94660dd4830e2f31c9baba90 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 27 Feb 2026 03:37:26 +0000 Subject: [PATCH 1/2] feat(opencode-adapter): handle ACP image content blocks in session/update The translate_session_update function only handled text content blocks from agent_message_chunk/agent_thought_chunk notifications, silently dropping image content. This adds support for ACP ImageContent blocks (type=image, uri, mimeType) by translating them into OpenCode "file" parts with the correct MIME type and file:// URI, matching the pattern used by the opencode_compat layer. Pending text is finalized before emitting the image part, and both are persisted for session replay. https://claude.ai/code/session_01Exgh67PQ5vVF1MxHDSHm4s --- server/packages/opencode-adapter/src/lib.rs | 140 +++++++++++++++----- 1 file changed, 109 insertions(+), 31 deletions(-) diff --git a/server/packages/opencode-adapter/src/lib.rs b/server/packages/opencode-adapter/src/lib.rs index b4e04d8a..9fa149f9 100644 --- a/server/packages/opencode-adapter/src/lib.rs +++ b/server/packages/opencode-adapter/src/lib.rs @@ -3968,41 +3968,119 @@ async fn translate_session_update( } match kind { - // ── Text / thought chunk ─────────────────────────────────────── + // ── Text / thought / image chunk ───────────────────────────── "agent_message_chunk" | "agent_thought_chunk" => { - // ContentChunk.content is a ContentBlock; for text it has { type: "text", text: "…" } - let chunk = update - .pointer("/content/text") + // ContentChunk.content is a ContentBlock discriminated by `type`. + let content_type = update + .pointer("/content/type") .and_then(Value::as_str) - .unwrap_or(""); - if chunk.is_empty() { - return; - } + .unwrap_or("text"); + + match content_type { + // ── Image content block ────────────────────────────────── + "image" => { + // ACP ImageContent: { type: "image", uri: "file:///…", mimeType: "image/png" } + let uri = update + .pointer("/content/uri") + .and_then(Value::as_str) + .unwrap_or(""); + if uri.is_empty() { + return; + } + let mime = update + .pointer("/content/mimeType") + .and_then(Value::as_str) + .unwrap_or("image/png"); - // Accumulate into a single part — reuse the same part ID so the - // UI updates in-place instead of creating a new line per chunk. - text_accum.push_str(chunk); - let part_id = text_part_id.get_or_insert_with(|| { - let id = format!("part_{message_id}_{part_counter}"); - *part_counter += 1; - id - }); - let part = json!({ - "id": *part_id, - "sessionID": session_id, - "messageID": message_id, - "type": "text", - "text": *text_accum, - }); - state.emit_event(json!({ - "type":"message.part.updated", - "properties":{ - "sessionID": session_id, - "messageID": message_id, - "part": part, - "delta": chunk + // Derive a filesystem path from the URI for the filename field. + let path = uri.strip_prefix("file://").unwrap_or(uri); + + // Finalize any accumulated text part before emitting the image. + if let Some(tid) = text_part_id.take() { + let part = json!({ + "id": tid, + "sessionID": session_id, + "messageID": message_id, + "type": "text", + "text": *text_accum, + }); + let env = json!({ + "jsonrpc":"2.0", + "method":"_sandboxagent/opencode/message", + "params":{"message":{"info":{"id": message_id},"parts":[part]}} + }); + if let Err(err) = state.persist_event(session_id, "agent", &env).await { + warn!(?err, "failed to persist ACP text part before image"); + } + text_accum.clear(); + } + + let part_id = format!("part_{message_id}_{part_counter}"); + *part_counter += 1; + let part = json!({ + "id": part_id, + "sessionID": session_id, + "messageID": message_id, + "type": "file", + "mime": mime, + "filename": path, + "url": uri, + }); + let env = json!({ + "jsonrpc":"2.0", + "method":"_sandboxagent/opencode/message", + "params":{"message":{"info":{"id": message_id},"parts":[part.clone()]}} + }); + if let Err(err) = state.persist_event(session_id, "agent", &env).await { + warn!(?err, "failed to persist ACP image part"); + } + state.emit_event(json!({ + "type":"message.part.updated", + "properties":{ + "sessionID": session_id, + "messageID": message_id, + "part": part + } + })); } - })); + + // ── Text content block (default) ──────────────────────── + _ => { + // For text: { type: "text", text: "…" } + let chunk = update + .pointer("/content/text") + .and_then(Value::as_str) + .unwrap_or(""); + if chunk.is_empty() { + return; + } + + // Accumulate into a single part — reuse the same part ID so the + // UI updates in-place instead of creating a new line per chunk. + text_accum.push_str(chunk); + let part_id = text_part_id.get_or_insert_with(|| { + let id = format!("part_{message_id}_{part_counter}"); + *part_counter += 1; + id + }); + let part = json!({ + "id": *part_id, + "sessionID": session_id, + "messageID": message_id, + "type": "text", + "text": *text_accum, + }); + state.emit_event(json!({ + "type":"message.part.updated", + "properties":{ + "sessionID": session_id, + "messageID": message_id, + "part": part, + "delta": chunk + } + })); + } + } } // ── Tool call initiation ─────────────────────────────────────── From 842aba30696579f25661cd24bab263a81af4f1f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 27 Feb 2026 04:44:21 +0000 Subject: [PATCH 2/2] feat(inspector): render image content blocks in chat UI Add image support to the inspector UI so that ACP agent_message_chunk notifications with type=image content blocks are displayed inline. - Add `image` kind and field to TimelineEntry type - Parse image content blocks in the App.tsx event-to-timeline mapper, converting file:// URIs to /v1/fs/file API URLs for browser rendering - Flush accumulated text before emitting an image entry - Render tags in ChatMessages for image timeline entries https://claude.ai/code/session_01Exgh67PQ5vVF1MxHDSHm4s --- frontend/packages/inspector/src/App.tsx | 21 +++++++++++++++++-- .../src/components/chat/ChatMessages.tsx | 10 +++++++-- .../inspector/src/components/chat/types.ts | 4 +++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/frontend/packages/inspector/src/App.tsx b/frontend/packages/inspector/src/App.tsx index 2dfe24dc..3f2eb5ca 100644 --- a/frontend/packages/inspector/src/App.tsx +++ b/frontend/packages/inspector/src/App.tsx @@ -996,8 +996,25 @@ export default function App() { switch (update.sessionUpdate) { case "agent_message_chunk": { - const content = update.content as { type?: string; text?: string } | undefined; - if (content?.type === "text" && content.text) { + const content = update.content as { type?: string; text?: string; uri?: string; mimeType?: string } | undefined; + if (content?.type === "image" && content.uri) { + // Flush any accumulated text before the image + flushAssistant(time); + // Convert file:// URIs to the /v1/fs/file API endpoint so the + // browser can fetch the image from the Sandbox Agent server. + const filePath = content.uri.startsWith("file://") + ? content.uri.slice("file://".length) + : content.uri; + const displayUri = `/v1/fs/file?path=${encodeURIComponent(filePath)}`; + entries.push({ + id: `image-${event.id}`, + eventId: event.id, + kind: "image", + time, + role: "assistant", + image: { uri: displayUri, mimeType: content.mimeType ?? "image/png" }, + }); + } else if (content?.type === "text" && content.text) { if (!assistantAccumId) { assistantAccumId = `assistant-${event.id}`; assistantAccumText = ""; diff --git a/frontend/packages/inspector/src/components/chat/ChatMessages.tsx b/frontend/packages/inspector/src/components/chat/ChatMessages.tsx index a96e7a57..6f1221eb 100644 --- a/frontend/packages/inspector/src/components/chat/ChatMessages.tsx +++ b/frontend/packages/inspector/src/components/chat/ChatMessages.tsx @@ -246,14 +246,20 @@ const ChatMessages = ({ return ; } - // Regular message + // Regular message or image const entry = group.entries[0]; const messageClass = getMessageClass(entry); return (
- {entry.text ? ( + {entry.image ? ( + Agent image + ) : entry.text ? ( ) : ( diff --git a/frontend/packages/inspector/src/components/chat/types.ts b/frontend/packages/inspector/src/components/chat/types.ts index a748b024..9fc48170 100644 --- a/frontend/packages/inspector/src/components/chat/types.ts +++ b/frontend/packages/inspector/src/components/chat/types.ts @@ -1,7 +1,7 @@ export type TimelineEntry = { id: string; eventId?: string; // Links back to the original event for navigation - kind: "message" | "tool" | "meta" | "reasoning"; + kind: "message" | "tool" | "meta" | "reasoning" | "image"; time: string; // For messages: role?: "user" | "assistant"; @@ -15,4 +15,6 @@ export type TimelineEntry = { reasoning?: { text: string; visibility?: string }; // For meta: meta?: { title: string; detail?: string; severity?: "info" | "error" }; + // For images: + image?: { uri: string; mimeType: string }; };