Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions e2e/fixtures/mock-ollama.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ const OLLAMA_PORT = 11434;
* @param {object} [opts]
* @param {string} [opts.chatReply='ok'] assistant content returned by /api/chat.
* @param {string[]} [opts.chatReplyQueue=[]] queue of replies to dequeue per call; falls back to chatReply when exhausted.
* @param {{status:number, message:string}} [opts.chatError] when set, /api/chat responds with this HTTP
* status and JSON body `{"error": message}` (the real Ollama 404 shape) instead of a reply. Off by
* default so existing specs are unaffected. chatCalls still increments so callers can assert it was hit.
* @param {string[]} [opts.installedModels=['gemma4:e2b-it-qat','llama3.2:3b']] models /api/tags reports as installed.
* @param {number} [opts.pullDelayMs=0] hold the /api/pull response open this long before completing it -- gives a
* cancel-mid-download test a real window to call cancel-pull before the mock would otherwise finish first.
Expand All @@ -31,6 +34,7 @@ const OLLAMA_PORT = 11434;
function startMockOllama(opts = {}) {
const chatReply = opts.chatReply ?? 'ok';
const chatReplyQueue = Array.isArray(opts.chatReplyQueue) ? [...opts.chatReplyQueue] : [];
const chatError = opts.chatError ?? null;
const installedModels = Array.isArray(opts.installedModels)
? opts.installedModels
: ['gemma4:e2b-it-qat', 'llama3.2:3b'];
Expand Down Expand Up @@ -111,6 +115,15 @@ function startMockOllama(opts = {}) {
const msgs = Array.isArray(body.messages) ? body.messages : [];
lastChatPrompt = msgs.length ? String(msgs[msgs.length - 1].content || '') : '';

// Opt-in failure mode: reply with the configured HTTP error (the real
// Ollama 404 "model not found" shape) so the summarizer's stream fails
// and must surface it rather than saving an empty summary.
if (chatError) {
res.writeHead(chatError.status, { 'content-type': 'application/json' });
res.end(JSON.stringify({ error: chatError.message }));
return;
}

// Dequeue reply or fall back to default
const reply = chatReplyQueue.length > 0 ? chatReplyQueue.shift() : chatReply;

Expand Down
89 changes: 89 additions & 0 deletions e2e/specs/summarize-failure.t2.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import { test, expect } from '../fixtures/electron';
import { realUserDataDir, fileSig } from '../fixtures/real-user-data';
import { writeUserConfig, writeMeetingSummary } from '../fixtures/user-config';
import { startMockOllama } from '../fixtures/mock-ollama';
import { readFileSync } from 'fs';

/**
* T2 — summarisation FAILURE contract (regression for the swallowed-stream bug /
* GH #301). Drives the real `reprocess` path (re-summarise an existing
* transcript — NO ASR, NO real model) with a mock Ollama that returns a 404
* "model not found" on /api/chat, and asserts the failure is surfaced honestly:
*
* 1. reprocess reports failure (success:false), NOT a false success.
* 2. the pre-existing summary on disk is byte-for-byte UNCHANGED — the failed
* stream must not clobber it with an empty summary.
*
* Before the fix the streaming generators swallowed the provider error and ended
* the stream silently, so the consumer saw an empty-but-"successful" stream, wrote
* an empty summary, printed STREAM_COMPLETE and exited 0. Model-free: the mock
* never loads a model, so this runs in the model-free t2 lane (no @pipeline tag).
*/

const TRANSCRIPT =
'Alice: we ship the release on Friday. Bob: the budget is fifty thousand dollars.';

const SEEDED_SUMMARY = 'PRE-EXISTING summary that a failed reprocess must NOT clobber';

type ReprocessResult = { success: boolean; error?: string };
type StenoWindow = Window & {
stenoai: {
meetings: {
reprocess: (summaryFile: string, regenTitle: boolean, name: string) => Promise<ReprocessResult>;
};
};
};

const readSummary = (file: string) => JSON.parse(readFileSync(file, 'utf8'));

test('reprocess surfaces a stream failure and leaves the existing summary untouched', async ({
launchApp,
userDataDir,
}) => {
const realDirBefore = fileSig(realUserDataDir());

// Local provider so the summarizer talks to the mock Ollama on 11434.
writeUserConfig(userDataDir, { ai_provider: 'local' });
const summaryFile = writeMeetingSummary(userDataDir, 'failure', {
name: 'Failure Meeting',
summary: SEEDED_SUMMARY,
key_points: ['seeded point'],
action_items: ['seeded action'],
transcript: TRANSCRIPT,
});
const summarySigBefore = fileSig(summaryFile);

// Mock Ollama answers /api/chat with the real Ollama 404 shape so the stream
// fails mid-flight instead of returning content.
const ollama = await startMockOllama({
chatError: { status: 404, message: "model 'gemma4:e2b-it-qat' not found" },
});
try {
const { page } = await launchApp();

const res = await page.evaluate(
(f) => (window as StenoWindow).stenoai.meetings.reprocess(f, false, 'Failure Meeting'),
summaryFile,
);

// (1) The operation must report failure, not a false success.
expect(res.success).toBe(false);

// The summariser must have actually attempted the call (proves we exercised
// the streaming path, not an earlier bail-out).
expect(ollama.chatCalls()).toBeGreaterThanOrEqual(1);

// (2) The pre-existing summary on disk is byte-for-byte unchanged — a failed
// stream must not overwrite it with an empty summary.
expect(fileSig(summaryFile)).toBe(summarySigBefore);
const after = readSummary(summaryFile);
expect(after.summary).toBe(SEEDED_SUMMARY);
expect(after.key_points).toEqual(['seeded point']);
expect(after.action_items).toEqual(['seeded action']);

// Keystone: the real user-data dir is byte-for-byte untouched.
expect(fileSig(realUserDataDir())).toBe(realDirBefore);
} finally {
await ollama.close();
}
});
50 changes: 33 additions & 17 deletions src/summarizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -853,9 +853,10 @@ def _adapter_stream(self, prompt: str, timeout_seconds: int = 600):
...
{"type": "done", "model": "...", "input_tokens": N, "output_tokens": M}
or {"type": "error", "error": "..."} on failure.
Yields the text portion of each chunk record. Errors are logged
and the generator returns silently — matches the streaming-error
behaviour of the cloud and ollama paths.
Yields the text portion of each chunk record. On any failure (an
error record, an HTTP error, or a transport error) it logs and then
RAISES so the consumer surfaces the real error via STREAM_ERROR —
matching the streaming-error contract of the cloud and ollama paths.
"""
import urllib.request
import urllib.error
Expand Down Expand Up @@ -890,16 +891,18 @@ def _adapter_stream(self, prompt: str, timeout_seconds: int = 600):
yield text
elif kind == "error":
logger.error(f"Adapter stream error: {record.get('error')}")
return
raise RuntimeError(f"Adapter stream error: {record.get('error')}")
elif kind == "done":
return
except urllib.error.HTTPError as e:
if e.code in (401, 403):
logger.error("Adapter stream rejected: session expired or unauthorized")
else:
logger.error(f"Adapter streaming failed: HTTP {e.code}")
raise
except Exception as e:
logger.error(f"Adapter streaming failed: {e}")
raise

def _anthropic_chat(self, prompt: str, timeout_seconds: int = 300) -> str:
"""Send a chat request via the Anthropic Messages API."""
Expand Down Expand Up @@ -1487,7 +1490,7 @@ def _stream_completion(self, prompt: str):
yield text
except Exception as e:
logger.error(f"Anthropic streaming failed: {e}")
return
raise
elif self.cloud_provider == "bedrock":
# Bedrock's /converse-stream uses amazon eventstream framing
# (binary, length-prefixed), which would need a dedicated
Expand All @@ -1503,7 +1506,7 @@ def _stream_completion(self, prompt: str):
yield text
except Exception as e:
logger.error(f"Bedrock summarisation failed: {e}")
return
raise
else:
try:
response = self.cloud_client.chat.completions.create(
Expand All @@ -1519,15 +1522,15 @@ def _stream_completion(self, prompt: str):
yield content
except Exception as e:
logger.error(f"OpenAI streaming failed: {e}")
return
raise
return

# Ollama (local or remote) — shared with the free-form template path.
try:
yield from self._stream_direct(prompt)
except Exception as e:
logger.error(f"Ollama streaming failed: {e}")
return
raise

def summarize_transcript_streaming(self, transcript: str, duration_minutes: int = 0, language: str = "en", notes: str = None, progress_callback=None, template_prompt: Optional[str] = None):
"""Generator that yields markdown chunks from the LLM.
Expand All @@ -1553,15 +1556,28 @@ def summarize_transcript_streaming(self, transcript: str, duration_minutes: int
# ACTIVE provider — not straight to Ollama, which has no client and
# would crash in cloud/adapter mode.
prompt = self._create_template_report_prompt(transcript, template_prompt, language, notes)
yield from self._stream_completion(prompt)
return

if self._needs_chunking(transcript, notes):
yield from self._map_reduce_streaming(transcript, language, notes, progress_callback)
return

prompt = self._create_markdown_prompt(transcript, language, notes)
yield from self._stream_completion(prompt)
inner = self._stream_completion(prompt)
empty_message = "Model returned an empty report"
elif self._needs_chunking(transcript, notes):
inner = self._map_reduce_streaming(transcript, language, notes, progress_callback)
empty_message = "Model returned an empty summary"
else:
prompt = self._create_markdown_prompt(transcript, language, notes)
inner = self._stream_completion(prompt)
empty_message = "Model returned an empty summary"

# Defense in depth mirroring _map_reduce_streaming's empty-reduce guard: a
# provider can complete the stream without raising yet yield nothing (or
# only whitespace). Route that through STREAM_ERROR instead of silently
# saving an empty summary/report. (_map_reduce_streaming already raises on
# an empty reduce, so wrapping it here is harmless.)
saw_content = False
for chunk in inner:
if chunk and chunk.strip():
saw_content = True
yield chunk
if not saw_content:
raise ValueError(empty_message)

def test_connection(self) -> bool:
"""
Expand Down
Loading
Loading