🐛 fix(sdk): web 抓取/搜索域九项修复——span OOM、解压预算、reader 零联网、全链路可中断

packages/sdk/src/tools/html-to-markdown.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { extractArticleMarkdown } from "./html-to-markdown";
+import { extractArticleMarkdown, normalizeTablesHtml } from "./html-to-markdown";
 
 describe("extractArticleMarkdown", () => {
   test("extracts article title and converts to markdown", () => {
@@ -78,3 +78,29 @@ describe("extractArticleMarkdown", () => {
     expect(result!.content.split("\n").length).toBeLessThan(250);
   });
 });
+
+describe("normalizeTablesHtml span budgets (#303)", () => {
+  test("absurd rowspan/colspan attributes are clamped and cannot explode the grid", () => {
+    const html = `<table><tr><td rowspan="99999999" colspan="99999999">x</td></tr></table>`;
+    const out = normalizeTablesHtml(html);
+    expect(out).toContain("x");
+    expect(out).not.toContain("rowspan");
+    expect(out.length).toBeLessThan(500_000); // clamped to a 100×100 grid, not millions of cells
+  });
+
+  test("a table whose clamped expansion exceeds the cell budget is left untouched", () => {
+    // 30 cells × (100×100) = 300,000 grid slots, past the 250k budget → skip normalization
+    const cells = Array.from({ length: 30 }, (_, i) => `<td rowspan="100" colspan="100">c${i}</td>`).join("");
+    const html = `<table><tr>${cells}</tr></table>`;
+    const out = normalizeTablesHtml(html);
+    expect(out).toContain('rowspan="100"');
+    expect(out).toContain('colspan="100"');
+  });
+
+  test("large-but-legal merged grids still expand", () => {
+    const html = `<table><tr><td rowspan="50" colspan="50">cell</td></tr></table>`;
+    const out = normalizeTablesHtml(html);
+    expect(out).not.toContain("rowspan");
+    expect(out).not.toContain("colspan");
+  });
+});

packages/sdk/src/tools/html-to-markdown.ts

@@ -42,18 +42,46 @@ function normalizeCell(cell: HTMLElement): void {
     .replace(/<\/p>\s*<p[^>]*>/gi, " ");
 }
 
-/**
- * Upper bound for rowspan/colspan expansion: the values come from remote HTML,
- * and an unbounded span would allocate a hostile number of grid cells (#303).
- * 100 keeps the worst-case grid (100×100 per merged cell) trivial to render;
- * real-world merged cells span single digits.
- */
+/** Per-cell upper bound for rowspan/colspan (HTML spec clamps similarly). */
 const MAX_SPAN = 100;
+/** Expanded-grid budget; tables beyond it skip normalization untouched. */
+const MAX_TABLE_GRID_CELLS = 250_000;
 
 function clampSpan(attribute: string | null): number {
   return Math.min(MAX_SPAN, Math.max(1, Number.parseInt(attribute ?? "1", 10) || 1));
 }
 
+/**
+ * Expand rowspan/colspan into a rectangular grid, or return null when the
+ * expansion would blow past the cell budget (hostile span attributes must not
+ * materialize millions of grid slots).
+ */
+function buildTableGrid(rows: HTMLElement[]): { grid: Array<Array<HTMLElement | undefined>>; primary: Set<string> } | null {
+  const grid: Array<Array<HTMLElement | undefined>> = [];
+  const primary = new Set<string>();
+  let placedCells = 0;
+  for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) {
+    const currentRow = grid[rowIndex] ??= [];
+    let colIndex = 0;
+    for (const cell of Array.from(rows[rowIndex]!.children).filter(
+      (child): child is HTMLElement => child.tagName === "TD" || child.tagName === "TH",
+    )) {
+      while (currentRow[colIndex]) colIndex++;
+      const rowSpan = clampSpan(cell.getAttribute("rowspan"));
+      const colSpan = clampSpan(cell.getAttribute("colspan"));
+      primary.add(`${rowIndex}:${colIndex}`);
+      placedCells += rowSpan * colSpan;
+      if (placedCells > MAX_TABLE_GRID_CELLS) return null;
+      for (let r = 0; r < rowSpan; r++) {
+        const targetRow = grid[rowIndex + r] ??= [];
+        for (let c = 0; c < colSpan; c++) targetRow[colIndex + c] ??= cell;
+      }
+      colIndex += colSpan;
+    }
+  }
+  return { grid, primary };
+}
+
 /**
  * Normalize HTML tables before Turndown. GFM has no representation for merged
  * cells, so colspan/rowspan are expanded into a deterministic rectangular grid
@@ -68,27 +96,12 @@ export function normalizeTablesHtml(html: string, url = "https://lume.invalid/")
     if (rows.length === 0) continue;
     for (const cell of Array.from(table.querySelectorAll("td,th"))) normalizeCell(cell as HTMLElement);
 
-    const grid: Array<Array<HTMLElement | undefined>> = [];
-    const primary = new Set<string>();
-    for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) {
-      const currentRow = grid[rowIndex] ??= [];
-      let colIndex = 0;
-      for (const cell of Array.from(rows[rowIndex]!.children).filter(
-        (child): child is HTMLElement => child.tagName === "TD" || child.tagName === "TH",
-      )) {
-        while (currentRow[colIndex]) colIndex++;
-        const rowSpan = clampSpan(cell.getAttribute("rowspan"));
-        const colSpan = clampSpan(cell.getAttribute("colspan"));
-        primary.add(`${rowIndex}:${colIndex}`);
-        for (let r = 0; r < rowSpan; r++) {
-          const targetRow = grid[rowIndex + r] ??= [];
-          for (let c = 0; c < colSpan; c++) targetRow[colIndex + c] ??= cell;
-        }
-        colIndex += colSpan;
-      }
-    }
+    const expanded = buildTableGrid(rows as HTMLElement[]);
+    if (!expanded) continue;
+    const { grid, primary } = expanded;
 
-    const width = Math.max(...grid.map(row => row.length));
+    let width = 0;
+    for (const row of grid) if (row.length > width) width = row.length;
     const normalizedRows = grid.map((row, rowIndex) => {
       const tr = doc.createElement("tr");
       for (let colIndex = 0; colIndex < width; colIndex++) {

packages/sdk/src/tools/image-pipeline.test.ts

@@ -135,4 +135,36 @@ describe("downloadAndLocalizeImages", () => {
     expect(out.failed).toBe(0);
     await tmp.rm(dir, { recursive: true, force: true });
   });
+
+  test("caps downloads at 50 images per page (#342)", async () => {
+    const tmp = await import("node:fs/promises");
+    const dir = await tmp.mkdtemp((await import("node:os")).tmpdir() + "/lume-img-");
+    const png1x1 = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC", "base64");
+    const fetched: string[] = [];
+    const fakeFetch = (async (url: string) => { fetched.push(url); return new Response(png1x1, { status: 200, headers: { "content-type": "image/png" } }); }) as any;
+    const html = Array.from({ length: 60 }, (_, i) => `<img src="https://example.com/i/${i}.png">`).join("");
+    const out = await downloadAndLocalizeImages(html, "https://example.com/page", dir, "download", fakeFetch);
+    expect(fetched.length).toBe(50);
+    expect(out.downloaded).toBe(50);
+    await tmp.rm(dir, { recursive: true, force: true });
+  });
+
+  test("stops downloading once the signal aborts mid-page (#342)", async () => {
+    const tmp = await import("node:fs/promises");
+    const dir = await tmp.mkdtemp((await import("node:os")).tmpdir() + "/lume-img-");
+    const png1x1 = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC", "base64");
+    const controller = new AbortController();
+    let calls = 0;
+    const fakeFetch = (async (url: string) => {
+      calls++;
+      if (calls === 3) controller.abort();
+      return new Response(png1x1, { status: 200, headers: { "content-type": "image/png" } });
+    }) as any;
+    const html = Array.from({ length: 20 }, (_, i) => `<img src="https://example.com/i/${i}.png">`).join("");
+    const out = await downloadAndLocalizeImages(html, "https://example.com/page", dir, "download", fakeFetch, undefined, controller.signal);
+    expect(calls).toBe(3);
+    expect(out.downloaded).toBe(2);
+    expect(out.failed).toBe(1);
+    await tmp.rm(dir, { recursive: true, force: true });
+  });
 });

packages/sdk/src/tools/image-pipeline.ts

@@ -59,7 +59,7 @@ function isPlaceholder(src: string): boolean {
   return src.startsWith("data:image/svg"); // common transparent placeholder
 }
 
-function sniffExt(contentType: string, url: string): string {
+export function sniffExt(contentType: string, url: string): string {
   const ct = contentType.toLowerCase();
   if (ct.includes("png")) return ".png";
   if (ct.includes("jpeg") || ct.includes("jpg")) return ".jpg";
@@ -94,10 +94,14 @@ export interface LocalizeResult {
   failed: number;
 }
 
+/** Upper bound on image downloads per page; a huge gallery must not stall a fetch. */
+const MAX_IMAGE_DOWNLOADS = 50;
+
 /**
  * Walk <img> in html: resolve lazy src, optionally download (Referer = page origin,
  * anti-hotlink), rewrite to lume-file:// local path. Runs BEFORE Readability/Turndown
- * so converted Markdown keeps working image links.
+ * so converted Markdown keeps working image links. Stops early when `signal`
+ * aborts or after MAX_IMAGE_DOWNLOADS downloads.
  */
 export async function downloadAndLocalizeImages(
   html: string,
@@ -106,6 +110,7 @@ export async function downloadAndLocalizeImages(
   mode: ImageMode,
   fetchImpl: FetchImpl,
   sandbox?: SandboxSettings,
+  signal?: AbortSignal,
 ): Promise<LocalizeResult> {
   if (mode === "off") return { html, downloaded: 0, failed: 0 };
 
@@ -115,12 +120,14 @@ export async function downloadAndLocalizeImages(
 
   let downloaded = 0;
   let failed = 0;
+  let downloadAttempts = 0;
 
   if (mode === "download") {
     try { await mkdir(imagesDir, { recursive: true }); } catch { /* ignore */ }
   }
 
   for (const img of Array.from(doc.querySelectorAll("img"))) {
+    if (signal?.aborted) break;
     const src = resolveImgSrc(img as unknown as HTMLImageElement);
     if (!src) continue;
     let absUrl: string;
@@ -133,6 +140,7 @@ export async function downloadAndLocalizeImages(
     }
 
     // mode === "download"
+    if (downloadAttempts >= MAX_IMAGE_DOWNLOADS) break;
     // Sandbox: only fetch images that are (a) same-origin as the page, (b) on the
     // whitelisted CDN host set, or (c) explicitly allowed by the network sandbox.
     // Otherwise skip the download (degrade to the original URL), do NOT throw.
@@ -142,11 +150,13 @@ export async function downloadAndLocalizeImages(
       failed++;
       continue;
     }
+    downloadAttempts++;
     try {
       const res = await loadBinary(absUrl, {
         fetchImpl,
         maxBytes: 20 * 1024 * 1024,
         timeoutMs: 15000,
+        signal,
         sandbox: ALLOWED_IMAGE_HOSTS.has(new URL(absUrl).hostname.toLowerCase()) ? undefined : sandbox,
         headers: {
           ...(origin ? { Referer: `${origin}/` } : {}),

packages/sdk/src/tools/web-fetch-content.test.ts

@@ -1,6 +1,18 @@
 import { describe, expect, test } from "bun:test";
 import { strToU8, zipSync } from "fflate";
-import { renderStructuredBinary } from "./web-fetch-content.js";
+import { renderStructuredBinary, zipEntries } from "./web-fetch-content.js";
+
+/** Overwrite the local + central directory uncompressed-size fields with a lie. */
+function lieAboutSizes(zip: Uint8Array, claimedSize: number): void {
+  const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
+  view.setUint32(22, claimedSize, true); // local file header: uncompressed size
+  for (let i = 0; i < zip.length - 4; i++) {
+    if (zip[i] === 0x50 && zip[i + 1] === 0x4b && zip[i + 2] === 0x01 && zip[i + 3] === 0x02) {
+      view.setUint32(i + 24, claimedSize, true); // central directory entry: uncompressed size
+      return;
+    }
+  }
+}
 
 describe("renderStructuredBinary", () => {
   test("lists archive files and readable text entries", async () => {
@@ -26,3 +38,19 @@ describe("renderStructuredBinary", () => {
     expect(result?.markdown).toContain("Hello DOCX");
   });
 });
+
+describe("zipEntries real-byte metering (#340)", () => {
+  test("measures actual inflated bytes instead of trusting central-directory sizes", () => {
+    const payload = new Uint8Array(256 * 1024); // zeros deflate to a few hundred bytes
+    const archive = zipSync({ "big.bin": payload });
+    lieAboutSizes(archive, 100);
+    const entries = zipEntries(archive);
+    expect(entries["big.bin"].byteLength).toBe(256 * 1024);
+  });
+
+  test("still extracts stored (method 0) entries", () => {
+    const archive = zipSync({ "note.txt": strToU8("plain stored text") }, { level: 0 });
+    const entries = zipEntries(archive);
+    expect(new TextDecoder().decode(entries["note.txt"])).toBe("plain stored text");
+  });
+});

packages/sdk/src/tools/web-fetch-content.ts

@@ -1,5 +1,5 @@
 import { createRequire } from "node:module";
-import { unzipSync } from "fflate";
+import { Unzip, UnzipInflate, UnzipPassThrough } from "fflate";
 import { createTurndown, extractArticleMarkdown } from "./html-to-markdown.js";
 
 const require = createRequire(import.meta.url);
@@ -43,23 +43,57 @@ function archiveText(name: string, bytes: Uint8Array): string | null {
   return text.length > 0 ? text.slice(0, MAX_TEXT_ENTRY_BYTES) : null;
 }
 
-function zipEntries(bytes: Uint8Array): Record<string, Uint8Array> {
-  // The filter runs before each entry is decompressed and reports the
-  // uncompressed size from the central directory, so entries beyond the
-  // entry/byte budget are never inflated into memory (zip bombs abort).
-  let entries = 0;
+function concatChunks(chunks: Uint8Array[]): Uint8Array {
+  const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return out;
+}
+
+export function zipEntries(bytes: Uint8Array): Record<string, Uint8Array> {
+  // Stream the archive and meter each entry's REAL inflated byte count via
+  // ondata. Central-directory size claims are untrusted — a lying header must
+  // not steer the buffer budget, and once the byte ceiling is hit we simply
+  // stop pushing compressed input, so the remaining entries are never
+  // inflated (zip bombs abort instead of spinning the CPU).
+  const entries = new Map<string, Uint8Array>();
+  let accepted = 0;
   let totalBytes = 0;
-  return unzipSync(bytes, {
-    filter: (file) => {
-      if (entries >= MAX_ARCHIVE_ENTRIES) return false;
-      if (totalBytes + file.originalSize > MAX_ARCHIVE_UNCOMPRESSED_BYTES) {
-        throw new Error(`zip archive uncompressed size exceeds ${MAX_ARCHIVE_UNCOMPRESSED_BYTES} bytes`);
+  let failure: string | null = null;
+
+  const unzip = new Unzip((file) => {
+    if (failure || accepted >= MAX_ARCHIVE_ENTRIES || !file.name || file.name.endsWith("/")) return;
+    accepted += 1;
+    const chunks: Uint8Array[] = [];
+    file.ondata = (err, data, final) => {
+      if (err) {
+        failure ??= err.message;
+        return;
       }
-      entries += 1;
-      totalBytes += file.originalSize;
-      return true;
-    },
-  }) as Record<string, Uint8Array>;
+      totalBytes += data.byteLength;
+      if (totalBytes > MAX_ARCHIVE_UNCOMPRESSED_BYTES) {
+        failure ??= `zip archive uncompressed size exceeds ${MAX_ARCHIVE_UNCOMPRESSED_BYTES} bytes`;
+        chunks.length = 0;
+        return;
+      }
+      chunks.push(data);
+      if (final) entries.set(file.name, concatChunks(chunks));
+    };
+    file.start();
+  });
+  unzip.register(UnzipInflate);
+  unzip.register(UnzipPassThrough);
+
+  const CHUNK_SIZE = 1 << 16;
+  for (let offset = 0; offset < bytes.length && !failure; offset += CHUNK_SIZE) {
+    unzip.push(bytes.subarray(offset, offset + CHUNK_SIZE), offset + CHUNK_SIZE >= bytes.length);
+  }
+  if (failure) throw new Error(failure);
+  return Object.fromEntries(entries);
 }
 
 function renderZip(bytes: Uint8Array, url: string, kind: string): StructuredBinaryResult {