fix(perf): strip oversized inline images at write time + covering index (prod beach-ball fix)

src/main/db/index.ts

@@ -19,6 +19,7 @@ import type {
 import { createLogger } from "../services/logger";
 import { parseAutoDraftTaskId, AUTO_DRAFT_TASK_ID_LIKE_PATTERN } from "../agents/task-id";
 import { runMigrations } from "./migrations";
+import { stripLargeDataUris } from "../../shared/body-sanitizer";
 
 const log = createLogger("db");
 
@@ -288,7 +289,12 @@ export function getAllEmailIds(accountId?: string): string[] {
 
 export function saveEmail(email: Email, accountId: string = "default"): void {
   const db = getDatabase();
-  const bodyText = stripHtmlForSearch(email.body);
+  // Strip oversized inline data: URIs at the write boundary. The renderer
+  // replaces them with placeholders before display anyway; storing them made
+  // the emails table ~30x larger than its useful content and turned every
+  // synchronous main-process scan into a multi-second freeze (see migration 8).
+  const body = stripLargeDataUris(email.body);
+  const bodyText = stripHtmlForSearch(body);
   const stmt = db.prepare(`
     INSERT OR REPLACE INTO emails (id, account_id, thread_id, subject, from_address, to_address, cc_address, bcc_address, body, body_text, snippet, date, fetched_at, label_ids, attachments, message_id, in_reply_to)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
@@ -302,7 +308,7 @@ export function saveEmail(email: Email, accountId: string = "default"): void {
     email.to,
     email.cc || null,
     email.bcc || null,
-    email.body,
+    body,
     bodyText,
     email.snippet || null,
     email.date,
@@ -1051,38 +1057,6 @@ export function isThreadFullyAnalyzed(threadId: string, accountId?: string): boo
   return row.unanalyzed === 0;
 }
 
-/**
- * Strip inline data: URIs larger than ~50KB from email HTML bodies.
- * These are typically multi-MB base64-encoded images or videos that bloat IPC
- * transfer, Zustand store memory, and DOM rendering in the renderer process.
- * The original bodies remain in the DB; this only affects what crosses IPC.
- */
-function stripLargeDataUris(body: string): string {
-  if (!body || !body.includes("data:")) return body;
-  // If the body is under 50KB total, no substring can exceed the 50KB data URI threshold
-  if (body.length < 50_000) return body;
-
-  return body.replace(
-    /(<img\b[^>]*?\bsrc\s*=\s*["'])(data:[^"']+)(["'][^>]*>)/gi,
-    (match, before: string, dataUri: string, after: string) => {
-      if (dataUri.length < 50_000) return match;
-      const mimeMatch = dataUri.match(/^data:([^;,]+)/);
-      const mime = mimeMatch?.[1] ?? "image";
-      const sizeKB = Math.round((dataUri.length * 3) / 4 / 1024);
-      const sizeLabel = sizeKB >= 1024 ? `${(sizeKB / 1024).toFixed(1)} MB` : `${sizeKB} KB`;
-      // Theme-neutral colors: the main process doesn't know the renderer's theme,
-      // so use mid-tone grays that are legible on both light and dark backgrounds.
-      const svg =
-        `<svg xmlns="http://www.w3.org/2000/svg" width="400" height="60">` +
-        `<rect width="400" height="60" rx="8" fill="#d1d5db"/>` +
-        `<text x="200" y="35" text-anchor="middle" fill="#4b5563" font-family="system-ui" font-size="13">` +
-        `Inline ${mime} (${sizeLabel}) — too large to display inline` +
-        `</text></svg>`;
-      return `${before}data:image/svg+xml,${encodeURIComponent(svg)}${after}`;
-    },
-  );
-}
-
 function rowToDashboardEmail(row: Record<string, unknown>): DashboardEmail {
   // Parse labelIds from JSON string if present
   let labelIds: string[] | undefined;
@@ -1114,6 +1088,8 @@ function rowToDashboardEmail(row: Record<string, unknown>): DashboardEmail {
     to: row.to as string,
     ...(row.cc ? { cc: row.cc as string } : {}),
     ...(row.bcc ? { bcc: row.bcc as string } : {}),
+    // Backstop: rows written by saveEmail / migration 8 are already stripped,
+    // but rows from older builds (pre-migration-8) may still hold full bodies.
     body: stripLargeDataUris(row.body as string),
     snippet: row.snippet as string | undefined,
     date: row.date as string,

src/main/db/migrations.ts

@@ -10,6 +10,7 @@
  */
 import type BetterSqlite3 from "better-sqlite3";
 import { createLogger } from "../services/logger";
+import { stripLargeDataUris, DATA_URI_STRIP_THRESHOLD } from "../../shared/body-sanitizer";
 
 const log = createLogger("db-migrations");
 
@@ -19,6 +20,12 @@ interface Migration {
   version: number;
   name: string;
   up: (db: DatabaseInstance) => void;
+  /**
+   * Run VACUUM after all pending migrations complete. VACUUM cannot run
+   * inside a transaction (each migration runs in one), so it's deferred to
+   * the end of runNumberedMigrations and executed at most once.
+   */
+  vacuumAfter?: boolean;
 }
 
 /**
@@ -329,8 +336,9 @@ export const NUMBERED_MIGRATIONS: Migration[] = [
     //
     // Guard on table existence: migrations run BEFORE the SCHEMA `CREATE TABLE`
     // statements in initDatabase, so on a fresh DB the `emails` table won't
-    // exist yet. SCHEMA itself includes this index (see schema.ts), so fresh
-    // DBs are still covered — this migration only matters for existing DBs.
+    // exist yet — this migration only matters for existing DBs. (Migration 8
+    // later replaces this index with the wider idx_emails_all_light, which is
+    // what schema.ts now creates for fresh DBs.)
     up: (db) => {
       const tableExists = db
         .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='emails'")
@@ -354,6 +362,81 @@ export const NUMBERED_MIGRATIONS: Migration[] = [
       }
     },
   },
+  {
+    version: 8,
+    name: "strip_large_data_uris_and_widen_merge_cover_index",
+    vacuumAfter: true,
+    // Prod forensics (July 2026): the emails table was 1.6GB for ~15k rows
+    // because inline images were stored as base64 data: URIs inside body HTML
+    // (avg 106KB/row, max 29MB) — content the renderer strips to a placeholder
+    // before display anyway. Because `body` is declared before label_ids/
+    // message_id/in_reply_to, every inbox/sent/search scan had to walk each
+    // row's overflow-page chain even when body wasn't selected, freezing the
+    // main process for 0.8-12s per query (better-sqlite3 is synchronous).
+    // saveEmail now strips at the write boundary; this migration strips the
+    // rows written before the fix and reclaims the space via vacuumAfter.
+    //
+    // The index change widens idx_emails_merge_cover (whose four columns are
+    // this index's prefix, so it's strictly superseded) into a covering index
+    // for getInboxEmails' allLight query — SELECT id, account_id, thread_id,
+    // message_id, in_reply_to, date, label_ids over every row of an account —
+    // so it's served from index pages without touching table rows at all.
+    //
+    // Guard on table existence: migrations run BEFORE SCHEMA on fresh DBs;
+    // schema.ts creates the new index for those.
+    up: (db) => {
+      const tableExists = db
+        .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='emails'")
+        .get();
+      if (!tableExists) return;
+
+      // Strip BEFORE building the index. Building idx_emails_all_light must read
+      // label_ids/message_id/in_reply_to, which are declared after body, so on
+      // the pre-strip table it walks every fat row's overflow-page chain — the
+      // exact cost this migration removes. Stripping first shrinks the table so
+      // the index build scans the small version.
+      //
+      // Two-pass strip: collect candidate ids first, then load one body at a
+      // time — better-sqlite3 can't run statements while an iterator is open,
+      // and .all() on the bodies would pull the whole 1.5GB into memory. LIKE is
+      // ASCII-case-insensitive in SQLite by default, so `DATA:` bodies are also
+      // selected here (the strip itself is case-insensitive too).
+      const fatRows = db
+        .prepare("SELECT id FROM emails WHERE LENGTH(body) >= ? AND body LIKE '%data:%'")
+        .all(DATA_URI_STRIP_THRESHOLD) as Array<{ id: string }>;
+
+      if (fatRows.length > 0) {
+        log.info(
+          { candidates: fatRows.length },
+          "One-time migration: stripping oversized inline images from stored email bodies — this may take a minute on large databases",
+        );
+        const selectBody = db.prepare("SELECT body FROM emails WHERE id = ?");
+        const updateBody = db.prepare("UPDATE emails SET body = ? WHERE id = ?");
+        let strippedCount = 0;
+        let reclaimedChars = 0;
+        for (const { id } of fatRows) {
+          const row = selectBody.get(id) as { body: string } | undefined;
+          if (!row?.body) continue;
+          const stripped = stripLargeDataUris(row.body);
+          if (stripped !== row.body) {
+            updateBody.run(stripped, id);
+            strippedCount++;
+            reclaimedChars += row.body.length - stripped.length;
+          }
+        }
+        log.info(
+          { stripped: strippedCount, reclaimedMB: Math.round(reclaimedChars / 1024 / 1024) },
+          "Inline-image strip migration complete",
+        );
+      }
+
+      db.exec("DROP INDEX IF EXISTS idx_emails_merge_cover");
+      db.exec(`
+        CREATE INDEX IF NOT EXISTS idx_emails_all_light
+          ON emails(account_id, thread_id, message_id, in_reply_to, date, label_ids, id);
+      `);
+    },
+  },
 ];
 
 function runNumberedMigrations(db: DatabaseInstance): void {
@@ -375,6 +458,7 @@ function runNumberedMigrations(db: DatabaseInstance): void {
     log.info({ version: 0 }, "Migration system initialized at baseline");
   }
 
+  let needsVacuum = false;
   for (const migration of NUMBERED_MIGRATIONS) {
     if (migration.version > currentVersion) {
       log.info({ version: migration.version, name: migration.name }, "Running numbered migration");
@@ -384,6 +468,51 @@ function runNumberedMigrations(db: DatabaseInstance): void {
       });
       runInTransaction();
       currentVersion = migration.version;
+      if (migration.vacuumAfter) needsVacuum = true;
     }
   }
+
+  maybeVacuum(db, needsVacuum);
+}
+
+/**
+ * VACUUM to reclaim freed pages after a migration.
+ *
+ * VACUUM can't run inside a transaction, so it happens here after all
+ * migrations commit. The decision is also self-healing: a migration bumps
+ * schema_version and frees pages in one committed transaction, but VACUUM runs
+ * separately — if the app is force-quit in that window, the flag is lost and the
+ * freed space would never be reclaimed. So we ALSO vacuum whenever the freelist
+ * is large (checked cheaply on every startup), which reclaims after any
+ * interruption and skips when there's nothing to reclaim.
+ *
+ * VACUUM is an optimization, not a correctness requirement: it needs transient
+ * temp space and can throw SQLITE_FULL on a near-full disk. Since it runs at
+ * module load before any window exists, an unhandled throw would crash startup
+ * (a boot loop) even though every migration already committed — so failures are
+ * logged and swallowed.
+ */
+function maybeVacuum(db: DatabaseInstance, migrationRequestedVacuum: boolean): void {
+  let shouldVacuum = migrationRequestedVacuum;
+  if (!shouldVacuum) {
+    const pageCount = db.pragma("page_count", { simple: true }) as number;
+    const freelist = db.pragma("freelist_count", { simple: true }) as number;
+    const pageSize = db.pragma("page_size", { simple: true }) as number;
+    // >20% of the file free and >50MB of reclaimable space — enough to be worth
+    // the rewrite, rare enough not to fire on normal fragmentation.
+    if (pageCount > 0 && freelist / pageCount > 0.2 && freelist * pageSize > 50 * 1024 * 1024) {
+      shouldVacuum = true;
+    }
+  }
+  if (!shouldVacuum) return;
+
+  log.info("Running VACUUM to reclaim freed database space");
+  const start = Date.now();
+  try {
+    db.exec("VACUUM");
+    log.info({ durationMs: Date.now() - start }, "VACUUM complete");
+  } catch (err) {
+    // Non-fatal: the DB is fully consistent without VACUUM; it just stays large.
+    log.warn({ err }, "VACUUM failed — continuing startup; space will be reclaimed on a later run");
+  }
 }

src/main/db/schema.ts

@@ -357,9 +357,11 @@ CREATE INDEX IF NOT EXISTS idx_draft_memories_last_voted ON draft_memories(last_
 CREATE INDEX IF NOT EXISTS idx_emails_thread ON emails(thread_id);
 CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date);
 CREATE INDEX IF NOT EXISTS idx_emails_account ON emails(account_id);
--- Covering index for buildMergeCache (see db/index.ts) — keeps the per-account
--- merge cache rebuild served from index pages instead of row lookups.
-CREATE INDEX IF NOT EXISTS idx_emails_merge_cover ON emails(account_id, thread_id, message_id, in_reply_to);
+-- Covering index for buildMergeCache AND getInboxEmails' allLight query (see
+-- db/index.ts) — serves both entirely from index pages instead of row lookups,
+-- which matters because table rows can be large (email bodies). Supersedes the
+-- former idx_emails_merge_cover (its four columns are this index's prefix).
+CREATE INDEX IF NOT EXISTS idx_emails_all_light ON emails(account_id, thread_id, message_id, in_reply_to, date, label_ids, id);
 CREATE INDEX IF NOT EXISTS idx_analyses_needs_reply ON analyses(needs_reply);
 CREATE INDEX IF NOT EXISTS idx_drafts_status ON drafts(status);
 CREATE INDEX IF NOT EXISTS idx_sent_to_address ON sent_emails(to_address);

src/main/services/gmail-client.ts

@@ -19,6 +19,7 @@ import type {
   SendAsAlias,
 } from "../../shared/types";
 import { getAccounts } from "../db";
+import { DATA_URI_STRIP_THRESHOLD, inlineImagePlaceholder } from "../../shared/body-sanitizer";
 import { getDataDir } from "../data-dir";
 import { extractEmail } from "../utils/address-formatting";
 import { createLogger } from "./logger";
@@ -142,6 +143,15 @@ export function isAuthError(error: unknown): boolean {
   return false;
 }
 
+/** An inline (cid:) image referenced by an email body. `size` is the decoded
+ *  byte count reported by Gmail, used to skip downloading oversized images. */
+interface InlineImageInfo {
+  mimeType: string;
+  data?: string;
+  attachmentId?: string;
+  size?: number;
+}
+
 export class GmailClient {
   private oauth2Client: OAuth2Client | null = null;
   private gmail: ReturnType<typeof google.gmail> | null = null;
@@ -738,10 +748,8 @@ export class GmailClient {
    * Collect inline image parts from MIME tree (parts with Content-ID headers).
    * Returns a map from Content-ID (without angle brackets) to image metadata.
    */
-  private collectInlineImages(
-    payload: gmail_v1.Schema$MessagePart,
-  ): Map<string, { mimeType: string; data?: string; attachmentId?: string }> {
-    const images = new Map<string, { mimeType: string; data?: string; attachmentId?: string }>();
+  private collectInlineImages(payload: gmail_v1.Schema$MessagePart): Map<string, InlineImageInfo> {
+    const images = new Map<string, InlineImageInfo>();
 
     const walk = (part: gmail_v1.Schema$MessagePart) => {
       const headers = part.headers || [];
@@ -754,6 +762,7 @@ export class GmailClient {
           mimeType: part.mimeType,
           data: part.body?.data ?? undefined,
           attachmentId: part.body?.attachmentId ?? undefined,
+          size: part.body?.size ?? undefined,
         });
       }
 
@@ -823,7 +832,7 @@ export class GmailClient {
    */
   private async resolveInlineImages(
     html: string,
-    inlineImages: Map<string, { mimeType: string; data?: string; attachmentId?: string }>,
+    inlineImages: Map<string, InlineImageInfo>,
     messageId: string,
   ): Promise<string> {
     if (inlineImages.size === 0) return html;
@@ -846,6 +855,22 @@ export class GmailClient {
         const imageInfo = inlineImages.get(cid);
         if (!imageInfo) return;
 
+        // Oversized images would be stripped to a placeholder by saveEmail
+        // anyway (see shared/body-sanitizer.ts) — resolve them to the
+        // placeholder directly so we never download multi-MB attachments just
+        // to discard them. The stored data URI is `data:<mime>;base64,<b64>`:
+        // base64 is 4/3 of the decoded size, plus the `data:...;base64,` prefix.
+        const estimatedDataUriLength = imageInfo.size
+          ? Math.ceil(imageInfo.size / 3) * 4 + imageInfo.mimeType.length + 13
+          : undefined;
+        if (estimatedDataUriLength && estimatedDataUriLength >= DATA_URI_STRIP_THRESHOLD) {
+          replacements.set(
+            `cid:${cid}`,
+            inlineImagePlaceholder(imageInfo.mimeType, estimatedDataUriLength),
+          );
+          return;
+        }
+
         let base64Data = imageInfo.data;
 
         // Fetch from Gmail API if data wasn't inline in the payload

src/renderer/services/email-body-cache.ts

@@ -1,4 +1,5 @@
 import DOMPurify from "dompurify";
+import { stripLargeDataUris as stripLargeDataUrisShared } from "../../shared/body-sanitizer";
 
 /**
  * Checks if content appears to be HTML.
@@ -73,29 +74,15 @@ export function hasRichBackground(html: string): boolean {
  * Exported so EmailDetail can strip once and pass the light body to both
  * splitQuotedContent and EmailBodyRenderer.
  */
-const MAX_DATA_URI_LEN = 50_000; // ~37KB decoded
-
 export function stripLargeDataUris(body: string, useLightMode = true): string {
-  if (!body.includes("data:")) return body;
-
-  return body.replace(
-    /(<img\b[^>]*?\bsrc\s*=\s*["'])(data:[^"']+)(["'][^>]*>)/gi,
-    (_match, before: string, dataUri: string, after: string) => {
-      if (dataUri.length < MAX_DATA_URI_LEN) return _match;
-      const mimeMatch = dataUri.match(/^data:([^;,]+)/);
-      const mime = mimeMatch?.[1] ?? "image";
-      const sizeKB = Math.round((dataUri.length * 3) / 4 / 1024);
-      const sizeLabel = sizeKB >= 1024 ? `${(sizeKB / 1024).toFixed(1)} MB` : `${sizeKB} KB`;
-      const fill = useLightMode ? "#f3f4f6" : "#374151";
-      const textFill = useLightMode ? "#6b7280" : "#9ca3af";
-      const svg =
-        `<svg xmlns="http://www.w3.org/2000/svg" width="400" height="60">` +
-        `<rect width="400" height="60" rx="8" fill="${fill}"/>` +
-        `<text x="200" y="35" text-anchor="middle" fill="${textFill}" font-family="system-ui" font-size="13">` +
-        `Inline ${mime} (${sizeLabel}) — too large to display inline` +
-        `</text></svg>`;
-      return `${before}data:image/svg+xml,${encodeURIComponent(svg)}${after}`;
-    },
+  // Delegates to the shared (linear, case-insensitive) matcher — see
+  // shared/body-sanitizer.ts — passing theme-aware placeholder colors so the
+  // display placeholder matches the current light/dark mode.
+  return stripLargeDataUrisShared(
+    body,
+    useLightMode
+      ? { fill: "#f3f4f6", textFill: "#6b7280" }
+      : { fill: "#374151", textFill: "#9ca3af" },
   );
 }