diff --git a/src/main/db/index.ts b/src/main/db/index.ts index 2e4a1614..6f9dc628 100644 --- a/src/main/db/index.ts +++ b/src/main/db/index.ts @@ -19,6 +19,7 @@ import type { import { createLogger } from "../services/logger"; import { parseAutoDraftTaskId, AUTO_DRAFT_TASK_ID_LIKE_PATTERN } from "../agents/task-id"; import { runMigrations } from "./migrations"; +import { stripLargeDataUris } from "../../shared/body-sanitizer"; const log = createLogger("db"); @@ -288,7 +289,12 @@ export function getAllEmailIds(accountId?: string): string[] { export function saveEmail(email: Email, accountId: string = "default"): void { const db = getDatabase(); - const bodyText = stripHtmlForSearch(email.body); + // Strip oversized inline data: URIs at the write boundary. The renderer + // replaces them with placeholders before display anyway; storing them made + // the emails table ~30x larger than its useful content and turned every + // synchronous main-process scan into a multi-second freeze (see migration 8). + const body = stripLargeDataUris(email.body); + const bodyText = stripHtmlForSearch(body); const stmt = db.prepare(` INSERT OR REPLACE INTO emails (id, account_id, thread_id, subject, from_address, to_address, cc_address, bcc_address, body, body_text, snippet, date, fetched_at, label_ids, attachments, message_id, in_reply_to) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -302,7 +308,7 @@ export function saveEmail(email: Email, accountId: string = "default"): void { email.to, email.cc || null, email.bcc || null, - email.body, + body, bodyText, email.snippet || null, email.date, @@ -1051,38 +1057,6 @@ export function isThreadFullyAnalyzed(threadId: string, accountId?: string): boo return row.unanalyzed === 0; } -/** - * Strip inline data: URIs larger than ~50KB from email HTML bodies. - * These are typically multi-MB base64-encoded images or videos that bloat IPC - * transfer, Zustand store memory, and DOM rendering in the renderer process. - * The original bodies remain in the DB; this only affects what crosses IPC. - */ -function stripLargeDataUris(body: string): string { - if (!body || !body.includes("data:")) return body; - // If the body is under 50KB total, no substring can exceed the 50KB data URI threshold - if (body.length < 50_000) return body; - - return body.replace( - /(]*?\bsrc\s*=\s*["'])(data:[^"']+)(["'][^>]*>)/gi, - (match, before: string, dataUri: string, after: string) => { - if (dataUri.length < 50_000) return match; - const mimeMatch = dataUri.match(/^data:([^;,]+)/); - const mime = mimeMatch?.[1] ?? "image"; - const sizeKB = Math.round((dataUri.length * 3) / 4 / 1024); - const sizeLabel = sizeKB >= 1024 ? `${(sizeKB / 1024).toFixed(1)} MB` : `${sizeKB} KB`; - // Theme-neutral colors: the main process doesn't know the renderer's theme, - // so use mid-tone grays that are legible on both light and dark backgrounds. - const svg = - `` + - `` + - `` + - `Inline ${mime} (${sizeLabel}) — too large to display inline` + - ``; - return `${before}data:image/svg+xml,${encodeURIComponent(svg)}${after}`; - }, - ); -} - function rowToDashboardEmail(row: Record): DashboardEmail { // Parse labelIds from JSON string if present let labelIds: string[] | undefined; @@ -1114,6 +1088,8 @@ function rowToDashboardEmail(row: Record): DashboardEmail { to: row.to as string, ...(row.cc ? { cc: row.cc as string } : {}), ...(row.bcc ? { bcc: row.bcc as string } : {}), + // Backstop: rows written by saveEmail / migration 8 are already stripped, + // but rows from older builds (pre-migration-8) may still hold full bodies. body: stripLargeDataUris(row.body as string), snippet: row.snippet as string | undefined, date: row.date as string, diff --git a/src/main/db/migrations.ts b/src/main/db/migrations.ts index c250c956..bf7b4b57 100644 --- a/src/main/db/migrations.ts +++ b/src/main/db/migrations.ts @@ -10,6 +10,7 @@ */ import type BetterSqlite3 from "better-sqlite3"; import { createLogger } from "../services/logger"; +import { stripLargeDataUris, DATA_URI_STRIP_THRESHOLD } from "../../shared/body-sanitizer"; const log = createLogger("db-migrations"); @@ -19,6 +20,12 @@ interface Migration { version: number; name: string; up: (db: DatabaseInstance) => void; + /** + * Run VACUUM after all pending migrations complete. VACUUM cannot run + * inside a transaction (each migration runs in one), so it's deferred to + * the end of runNumberedMigrations and executed at most once. + */ + vacuumAfter?: boolean; } /** @@ -329,8 +336,9 @@ export const NUMBERED_MIGRATIONS: Migration[] = [ // // Guard on table existence: migrations run BEFORE the SCHEMA `CREATE TABLE` // statements in initDatabase, so on a fresh DB the `emails` table won't - // exist yet. SCHEMA itself includes this index (see schema.ts), so fresh - // DBs are still covered — this migration only matters for existing DBs. + // exist yet — this migration only matters for existing DBs. (Migration 8 + // later replaces this index with the wider idx_emails_all_light, which is + // what schema.ts now creates for fresh DBs.) up: (db) => { const tableExists = db .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='emails'") @@ -354,6 +362,81 @@ export const NUMBERED_MIGRATIONS: Migration[] = [ } }, }, + { + version: 8, + name: "strip_large_data_uris_and_widen_merge_cover_index", + vacuumAfter: true, + // Prod forensics (July 2026): the emails table was 1.6GB for ~15k rows + // because inline images were stored as base64 data: URIs inside body HTML + // (avg 106KB/row, max 29MB) — content the renderer strips to a placeholder + // before display anyway. Because `body` is declared before label_ids/ + // message_id/in_reply_to, every inbox/sent/search scan had to walk each + // row's overflow-page chain even when body wasn't selected, freezing the + // main process for 0.8-12s per query (better-sqlite3 is synchronous). + // saveEmail now strips at the write boundary; this migration strips the + // rows written before the fix and reclaims the space via vacuumAfter. + // + // The index change widens idx_emails_merge_cover (whose four columns are + // this index's prefix, so it's strictly superseded) into a covering index + // for getInboxEmails' allLight query — SELECT id, account_id, thread_id, + // message_id, in_reply_to, date, label_ids over every row of an account — + // so it's served from index pages without touching table rows at all. + // + // Guard on table existence: migrations run BEFORE SCHEMA on fresh DBs; + // schema.ts creates the new index for those. + up: (db) => { + const tableExists = db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='emails'") + .get(); + if (!tableExists) return; + + // Strip BEFORE building the index. Building idx_emails_all_light must read + // label_ids/message_id/in_reply_to, which are declared after body, so on + // the pre-strip table it walks every fat row's overflow-page chain — the + // exact cost this migration removes. Stripping first shrinks the table so + // the index build scans the small version. + // + // Two-pass strip: collect candidate ids first, then load one body at a + // time — better-sqlite3 can't run statements while an iterator is open, + // and .all() on the bodies would pull the whole 1.5GB into memory. LIKE is + // ASCII-case-insensitive in SQLite by default, so `DATA:` bodies are also + // selected here (the strip itself is case-insensitive too). + const fatRows = db + .prepare("SELECT id FROM emails WHERE LENGTH(body) >= ? AND body LIKE '%data:%'") + .all(DATA_URI_STRIP_THRESHOLD) as Array<{ id: string }>; + + if (fatRows.length > 0) { + log.info( + { candidates: fatRows.length }, + "One-time migration: stripping oversized inline images from stored email bodies — this may take a minute on large databases", + ); + const selectBody = db.prepare("SELECT body FROM emails WHERE id = ?"); + const updateBody = db.prepare("UPDATE emails SET body = ? WHERE id = ?"); + let strippedCount = 0; + let reclaimedChars = 0; + for (const { id } of fatRows) { + const row = selectBody.get(id) as { body: string } | undefined; + if (!row?.body) continue; + const stripped = stripLargeDataUris(row.body); + if (stripped !== row.body) { + updateBody.run(stripped, id); + strippedCount++; + reclaimedChars += row.body.length - stripped.length; + } + } + log.info( + { stripped: strippedCount, reclaimedMB: Math.round(reclaimedChars / 1024 / 1024) }, + "Inline-image strip migration complete", + ); + } + + db.exec("DROP INDEX IF EXISTS idx_emails_merge_cover"); + db.exec(` + CREATE INDEX IF NOT EXISTS idx_emails_all_light + ON emails(account_id, thread_id, message_id, in_reply_to, date, label_ids, id); + `); + }, + }, ]; function runNumberedMigrations(db: DatabaseInstance): void { @@ -375,6 +458,7 @@ function runNumberedMigrations(db: DatabaseInstance): void { log.info({ version: 0 }, "Migration system initialized at baseline"); } + let needsVacuum = false; for (const migration of NUMBERED_MIGRATIONS) { if (migration.version > currentVersion) { log.info({ version: migration.version, name: migration.name }, "Running numbered migration"); @@ -384,6 +468,51 @@ function runNumberedMigrations(db: DatabaseInstance): void { }); runInTransaction(); currentVersion = migration.version; + if (migration.vacuumAfter) needsVacuum = true; } } + + maybeVacuum(db, needsVacuum); +} + +/** + * VACUUM to reclaim freed pages after a migration. + * + * VACUUM can't run inside a transaction, so it happens here after all + * migrations commit. The decision is also self-healing: a migration bumps + * schema_version and frees pages in one committed transaction, but VACUUM runs + * separately — if the app is force-quit in that window, the flag is lost and the + * freed space would never be reclaimed. So we ALSO vacuum whenever the freelist + * is large (checked cheaply on every startup), which reclaims after any + * interruption and skips when there's nothing to reclaim. + * + * VACUUM is an optimization, not a correctness requirement: it needs transient + * temp space and can throw SQLITE_FULL on a near-full disk. Since it runs at + * module load before any window exists, an unhandled throw would crash startup + * (a boot loop) even though every migration already committed — so failures are + * logged and swallowed. + */ +function maybeVacuum(db: DatabaseInstance, migrationRequestedVacuum: boolean): void { + let shouldVacuum = migrationRequestedVacuum; + if (!shouldVacuum) { + const pageCount = db.pragma("page_count", { simple: true }) as number; + const freelist = db.pragma("freelist_count", { simple: true }) as number; + const pageSize = db.pragma("page_size", { simple: true }) as number; + // >20% of the file free and >50MB of reclaimable space — enough to be worth + // the rewrite, rare enough not to fire on normal fragmentation. + if (pageCount > 0 && freelist / pageCount > 0.2 && freelist * pageSize > 50 * 1024 * 1024) { + shouldVacuum = true; + } + } + if (!shouldVacuum) return; + + log.info("Running VACUUM to reclaim freed database space"); + const start = Date.now(); + try { + db.exec("VACUUM"); + log.info({ durationMs: Date.now() - start }, "VACUUM complete"); + } catch (err) { + // Non-fatal: the DB is fully consistent without VACUUM; it just stays large. + log.warn({ err }, "VACUUM failed — continuing startup; space will be reclaimed on a later run"); + } } diff --git a/src/main/db/schema.ts b/src/main/db/schema.ts index 856585e1..e7b3b98d 100644 --- a/src/main/db/schema.ts +++ b/src/main/db/schema.ts @@ -357,9 +357,11 @@ CREATE INDEX IF NOT EXISTS idx_draft_memories_last_voted ON draft_memories(last_ CREATE INDEX IF NOT EXISTS idx_emails_thread ON emails(thread_id); CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date); CREATE INDEX IF NOT EXISTS idx_emails_account ON emails(account_id); --- Covering index for buildMergeCache (see db/index.ts) — keeps the per-account --- merge cache rebuild served from index pages instead of row lookups. -CREATE INDEX IF NOT EXISTS idx_emails_merge_cover ON emails(account_id, thread_id, message_id, in_reply_to); +-- Covering index for buildMergeCache AND getInboxEmails' allLight query (see +-- db/index.ts) — serves both entirely from index pages instead of row lookups, +-- which matters because table rows can be large (email bodies). Supersedes the +-- former idx_emails_merge_cover (its four columns are this index's prefix). +CREATE INDEX IF NOT EXISTS idx_emails_all_light ON emails(account_id, thread_id, message_id, in_reply_to, date, label_ids, id); CREATE INDEX IF NOT EXISTS idx_analyses_needs_reply ON analyses(needs_reply); CREATE INDEX IF NOT EXISTS idx_drafts_status ON drafts(status); CREATE INDEX IF NOT EXISTS idx_sent_to_address ON sent_emails(to_address); diff --git a/src/main/services/gmail-client.ts b/src/main/services/gmail-client.ts index 6322c449..c324e8b3 100644 --- a/src/main/services/gmail-client.ts +++ b/src/main/services/gmail-client.ts @@ -19,6 +19,7 @@ import type { SendAsAlias, } from "../../shared/types"; import { getAccounts } from "../db"; +import { DATA_URI_STRIP_THRESHOLD, inlineImagePlaceholder } from "../../shared/body-sanitizer"; import { getDataDir } from "../data-dir"; import { extractEmail } from "../utils/address-formatting"; import { createLogger } from "./logger"; @@ -142,6 +143,15 @@ export function isAuthError(error: unknown): boolean { return false; } +/** An inline (cid:) image referenced by an email body. `size` is the decoded + * byte count reported by Gmail, used to skip downloading oversized images. */ +interface InlineImageInfo { + mimeType: string; + data?: string; + attachmentId?: string; + size?: number; +} + export class GmailClient { private oauth2Client: OAuth2Client | null = null; private gmail: ReturnType | null = null; @@ -738,10 +748,8 @@ export class GmailClient { * Collect inline image parts from MIME tree (parts with Content-ID headers). * Returns a map from Content-ID (without angle brackets) to image metadata. */ - private collectInlineImages( - payload: gmail_v1.Schema$MessagePart, - ): Map { - const images = new Map(); + private collectInlineImages(payload: gmail_v1.Schema$MessagePart): Map { + const images = new Map(); const walk = (part: gmail_v1.Schema$MessagePart) => { const headers = part.headers || []; @@ -754,6 +762,7 @@ export class GmailClient { mimeType: part.mimeType, data: part.body?.data ?? undefined, attachmentId: part.body?.attachmentId ?? undefined, + size: part.body?.size ?? undefined, }); } @@ -823,7 +832,7 @@ export class GmailClient { */ private async resolveInlineImages( html: string, - inlineImages: Map, + inlineImages: Map, messageId: string, ): Promise { if (inlineImages.size === 0) return html; @@ -846,6 +855,22 @@ export class GmailClient { const imageInfo = inlineImages.get(cid); if (!imageInfo) return; + // Oversized images would be stripped to a placeholder by saveEmail + // anyway (see shared/body-sanitizer.ts) — resolve them to the + // placeholder directly so we never download multi-MB attachments just + // to discard them. The stored data URI is `data:;base64,`: + // base64 is 4/3 of the decoded size, plus the `data:...;base64,` prefix. + const estimatedDataUriLength = imageInfo.size + ? Math.ceil(imageInfo.size / 3) * 4 + imageInfo.mimeType.length + 13 + : undefined; + if (estimatedDataUriLength && estimatedDataUriLength >= DATA_URI_STRIP_THRESHOLD) { + replacements.set( + `cid:${cid}`, + inlineImagePlaceholder(imageInfo.mimeType, estimatedDataUriLength), + ); + return; + } + let base64Data = imageInfo.data; // Fetch from Gmail API if data wasn't inline in the payload diff --git a/src/renderer/services/email-body-cache.ts b/src/renderer/services/email-body-cache.ts index 979a82ca..a372041a 100644 --- a/src/renderer/services/email-body-cache.ts +++ b/src/renderer/services/email-body-cache.ts @@ -1,4 +1,5 @@ import DOMPurify from "dompurify"; +import { stripLargeDataUris as stripLargeDataUrisShared } from "../../shared/body-sanitizer"; /** * Checks if content appears to be HTML. @@ -73,29 +74,15 @@ export function hasRichBackground(html: string): boolean { * Exported so EmailDetail can strip once and pass the light body to both * splitQuotedContent and EmailBodyRenderer. */ -const MAX_DATA_URI_LEN = 50_000; // ~37KB decoded - export function stripLargeDataUris(body: string, useLightMode = true): string { - if (!body.includes("data:")) return body; - - return body.replace( - /(]*?\bsrc\s*=\s*["'])(data:[^"']+)(["'][^>]*>)/gi, - (_match, before: string, dataUri: string, after: string) => { - if (dataUri.length < MAX_DATA_URI_LEN) return _match; - const mimeMatch = dataUri.match(/^data:([^;,]+)/); - const mime = mimeMatch?.[1] ?? "image"; - const sizeKB = Math.round((dataUri.length * 3) / 4 / 1024); - const sizeLabel = sizeKB >= 1024 ? `${(sizeKB / 1024).toFixed(1)} MB` : `${sizeKB} KB`; - const fill = useLightMode ? "#f3f4f6" : "#374151"; - const textFill = useLightMode ? "#6b7280" : "#9ca3af"; - const svg = - `` + - `` + - `` + - `Inline ${mime} (${sizeLabel}) — too large to display inline` + - ``; - return `${before}data:image/svg+xml,${encodeURIComponent(svg)}${after}`; - }, + // Delegates to the shared (linear, case-insensitive) matcher — see + // shared/body-sanitizer.ts — passing theme-aware placeholder colors so the + // display placeholder matches the current light/dark mode. + return stripLargeDataUrisShared( + body, + useLightMode + ? { fill: "#f3f4f6", textFill: "#6b7280" } + : { fill: "#374151", textFill: "#9ca3af" }, ); } diff --git a/src/shared/body-sanitizer.ts b/src/shared/body-sanitizer.ts new file mode 100644 index 00000000..0b888748 --- /dev/null +++ b/src/shared/body-sanitizer.ts @@ -0,0 +1,88 @@ +/** + * Strip oversized inline `data:` URIs from email HTML bodies. + * + * Prod forensics (July 2026) found 1.54GB of the 1.6GB emails table was + * base64 image payloads inlined into body HTML (avg 106KB/row, max 29MB) — + * content the renderer replaces with a placeholder before display anyway, so + * storing it only bloats every synchronous main-process query into a beach + * ball. Bodies are stripped once at the write boundary (saveEmail + migration + * 8) and again on read as a backstop for rows written by older builds. + * + * Lives in `shared/` (not `main/`) so the main process, the DB migration + * runner, and the renderer all import the SAME threshold, regex, and matcher — + * a drift between them would mean the two processes disagree about which images + * get stripped. This module must stay dependency-free (no electron, no + * data-dir): it is imported by db/migrations.ts, which runs in non-Electron + * test contexts, and by the renderer bundle. + */ + +/** Data URIs at or above this length (chars) are replaced with a placeholder. */ +export const DATA_URI_STRIP_THRESHOLD = 50_000; + +/** + * Matches a `src="data:..."` attribute value (any quote style, any element). + * + * Anchored on `src` rather than on `` so the scan is LINEAR: a lazy + * `]*?` prefix restarts at every ``, degrades to O(n²) — a sender-triggerable freeze + * of the synchronous main process (the exact beach ball this code prevents). + * `\bsrc\s*=\s*` only engages where "src=" actually appears, and `[^"']+` scans + * each value once with no nested quantifier, so total work is O(body length). + * + * Anchoring on `src` also broadens coverage beyond `` to `