feat(chat): summarize attachments + recordings knowledge base #431
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| import { useState } from 'react'; | ||
| import { MediaAttachment } from '../../types'; | ||
| import { transcriptSummarizer } from '../../services'; | ||
| import { useChatStore, useAppStore } from '../../stores'; | ||
| import logger from '../../utils/logger'; | ||
|
|
||
| /** Throttle for streaming the summary into the message (~20 paints/sec). */ | ||
| const STREAM_FLUSH_MS = 50; | ||
|
|
||
| /** mm:ss for a millisecond offset, used to label an attached transcript range. */ | ||
| function fmtClock(ms: number): string { | ||
| const total = Math.floor(ms / 1000); | ||
| const m = Math.floor(total / 60); | ||
| const s = total % 60; | ||
| return `${m}:${s.toString().padStart(2, '0')}`; | ||
| } | ||
|
|
||
| /** | ||
| * Summarize an attached document/transcript that is too large to fit the model's | ||
| * context window. Posts a user message ("Summarize <file>") and an assistant | ||
| * message, then streams progress into that assistant message (part i of N, | ||
| * combining) before replacing it with the final summary. Self-contained: reads | ||
| * the active conversation + model from the global stores, so it does not need | ||
| * props threaded down from the chat screen. | ||
| */ | ||
| export function useSummarizeAttachment() { | ||
| const [summarizingId, setSummarizingId] = useState<string | null>(null); | ||
|
|
||
| const handleSummarize = async (attachment: MediaAttachment): Promise<void> => { | ||
| if (summarizingId) return; | ||
| const text = attachment.textContent?.trim(); | ||
| if (!text) return; | ||
|
|
||
| const chat = useChatStore.getState(); | ||
| let conversationId = chat.activeConversationId; | ||
| if (!conversationId) { | ||
| const modelId = useAppStore.getState().activeModelId; | ||
| if (!modelId) return; // no model loaded - nothing to summarize with | ||
| conversationId = chat.createConversation(modelId); | ||
| chat.setActiveConversation(conversationId); | ||
| } | ||
|
|
||
| const label = attachment.fileName || 'transcript'; | ||
| const range = | ||
| attachment.transcriptStartMs != null && attachment.transcriptEndMs != null | ||
| ? ` (${fmtClock(attachment.transcriptStartMs)} to ${fmtClock(attachment.transcriptEndMs)})` | ||
| : ''; | ||
| chat.addMessage(conversationId, { role: 'user', content: `Summarize ${label}${range}` }); | ||
| const placeholder = chat.addMessage(conversationId, { role: 'assistant', content: 'Starting...' }); | ||
|
|
||
| setSummarizingId(attachment.id); | ||
| // Stream the work in place. The map phase streams each part as it is written | ||
| // (so a multi-chunk run shows text from part 1, not a static counter for | ||
| // minutes), then the final combine pass restreams the answer over the top. | ||
| // updateMessageContent rebuilds the conversations tree on every call, so we | ||
| // flush on a ~50ms timer (matching the main generation loop) rather than per | ||
| // token, otherwise the JS thread saturates and the UI only paints at the end. | ||
| let uiPhase: 'map' | 'final' = 'map'; | ||
| let total = 0; | ||
| let current = 0; | ||
| const doneParts: string[] = []; | ||
| let curPart = ''; | ||
| let finalText = ''; | ||
| let flushTimer: ReturnType<typeof setTimeout> | null = null; | ||
|
|
||
| const compose = (): string => { | ||
| if (uiPhase === 'final') return finalText || 'Combining the parts...'; | ||
| const parts = [...doneParts, curPart].filter((s) => s.trim()); | ||
| const header = total > 1 ? `Summarizing part ${current} of ${total}\n\n` : 'Summarizing...\n\n'; | ||
| return parts.length ? header + parts.join('\n\n') : header.trim(); | ||
| }; | ||
| const flush = () => { | ||
| flushTimer = null; | ||
| useChatStore.getState().updateMessageContent(conversationId!, placeholder.id, compose()); | ||
| }; | ||
| const scheduleFlush = () => { if (!flushTimer) flushTimer = setTimeout(flush, STREAM_FLUSH_MS); }; | ||
|
|
||
| try { | ||
| const summary = await transcriptSummarizer.summarize(text, { | ||
| onProgress: (p) => { | ||
| if (p.phase === 'chunking') { | ||
| total = p.total; | ||
| } else if (p.phase === 'mapping') { | ||
| if (p.total <= 1) { | ||
| uiPhase = 'final'; // single pass: the streamed text is the answer | ||
| } else { | ||
| if (curPart.trim()) doneParts.push(curPart.trim()); | ||
| curPart = ''; | ||
| total = p.total; | ||
| current = p.current; | ||
| } | ||
| } else if (p.phase === 'combining') { | ||
| if (curPart.trim()) doneParts.push(curPart.trim()); | ||
| curPart = ''; | ||
| uiPhase = 'final'; | ||
| finalText = ''; | ||
| } | ||
| scheduleFlush(); | ||
| }, | ||
| onToken: (delta) => { | ||
| if (uiPhase === 'final') finalText += delta; | ||
| else curPart += delta; | ||
| scheduleFlush(); | ||
| }, | ||
| }); | ||
| if (flushTimer) clearTimeout(flushTimer); | ||
| // Final trimmed summary (streamed text may have leading/trailing space). | ||
| useChatStore.getState().updateMessageContent(conversationId, placeholder.id, summary); | ||
| } catch (e) { | ||
| if (flushTimer) clearTimeout(flushTimer); | ||
| const msg = e instanceof Error ? e.message : 'Summarization failed'; | ||
| useChatStore.getState().updateMessageContent( | ||
| conversationId, | ||
| placeholder.id, | ||
| `Could not summarize this transcript.\n\n${msg}`, | ||
| ); | ||
| logger.warn('[useSummarizeAttachment] failed:', e); | ||
| } finally { | ||
| setSummarizingId(null); | ||
| } | ||
| }; | ||
|
|
||
| return { summarizingId, handleSummarize }; | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /** | ||
| * Chat Attachment Inbox | ||
| * | ||
| * A one-shot hand-off for seeding the chat composer with an attachment created | ||
| * elsewhere (e.g. the Pro recorder's "Attach to chat", which builds a transcript | ||
| * document and navigates to the Chat screen). The composer consumes the pending | ||
| * attachments once on mount, then clears them. | ||
| * | ||
| * Kept as a tiny module-level store (not a route param) so a large transcript | ||
| * body never has to be serialized through navigation, and so Pro can hand off to | ||
| * core without core importing anything from Pro. | ||
| */ | ||
| import { MediaAttachment } from '../types'; | ||
|
|
||
| let pending: MediaAttachment[] = []; | ||
|
|
||
| /** Queue attachments to seed the next chat composer mount. Replaces any pending. */ | ||
| export function setPendingChatAttachments(attachments: MediaAttachment[]): void { | ||
| pending = attachments; | ||
| } | ||
|
|
||
| /** Return and clear the pending attachments (empty array if none). */ | ||
| export function takePendingChatAttachments(): MediaAttachment[] { | ||
| const taken = pending; | ||
| pending = []; | ||
| return taken; | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🎯 Functional Correctness | 🟠 Major | ⚡ Quick win
Only remove the attachment after a successful summary.
handleSummarize()catches its own failures and also returns early on some no-op paths, so this unconditionalremoveAttachment()drops the original attachment even when summarization failed or never started.Suggested fix
const { summarizingId, handleSummarize } = useSummarizeAttachment(); const onSummarizeAttachment = async (attachment: MediaAttachment) => { - await handleSummarize(attachment); - removeAttachment(attachment.id); + const summarized = await handleSummarize(attachment); + if (summarized) removeAttachment(attachment.id); };handleSummarize()should return a success flag (or rethrow on failure) so the caller can make this decision correctly.🤖 Prompt for AI Agents