fix: keep ingest chunks within boundary budget

src/ingest-queue.ts

@@ -206,20 +206,23 @@ function splitIntoChunks(text: string, maxTokens: number): Array<{ text: string;
   while (offset < text.length) {
     let end = Math.min(offset + maxChars, text.length);
 
-    // Walk back up to 256 chars looking for a sentence boundary
+    // Walk back up to 256 chars looking for a sentence boundary.
+    // `end` is exclusive, so probes must stay inside [offset, end).
     const probeLimit = Math.min(256, end - offset);
     let hardCut = end;
     for (let i = 0; i < probeLimit; i++) {
-      const pos = end - i;
+      const pos = end - 1 - i;
+      if (pos <= offset) break;
       const ch = text.charAt(pos);
-      if (ch === "\n" && text.charAt(pos + 1) === "\n") {
+      if (ch === "\n" && pos + 1 < end && text.charAt(pos + 1) === "\n") {
         hardCut = pos + 2;
         break;
       }
     }
     if (hardCut === end) {
       for (let i = 0; i < probeLimit; i++) {
-        const pos = end - i;
+        const pos = end - 1 - i;
+        if (pos <= offset) break;
         if (text.charAt(pos) === "\n") {
           hardCut = pos + 1;
           break;
@@ -228,7 +231,8 @@ function splitIntoChunks(text: string, maxTokens: number): Array<{ text: string;
     }
     if (hardCut === end) {
       for (let i = 0; i < probeLimit; i++) {
-        const pos = end - i;
+        const pos = end - 1 - i;
+        if (pos <= offset) break;
         if (text.charAt(pos) === " ") {
           hardCut = pos;
           break;

test/unit/ingest-queue.test.ts

@@ -76,6 +76,31 @@ test("chunked ingest replaces first chunk then appends remaining chunks", async
   }
 });
 
+test("chunked ingest does not extend chunks past the configured budget at newline boundaries", async () => {
+  for (const text of [
+    `${"a".repeat(12)}\n${"b".repeat(10)}`,
+    `${"a".repeat(12)}\n\n${"b".repeat(10)}`,
+  ]) {
+    const calls: Array<{ mode?: IngestMode; text: string }> = [];
+    const queue = new IngestQueue(
+      async (params) => {
+        calls.push({ mode: params.mode, text: params.text });
+        return { ok: true };
+      },
+      async () => {},
+      { error() {}, warn() {} },
+      { chunkTokens: 3, maxRetries: 0 },
+    );
+
+    await queue.enqueueIngest("/vault/daily.md", text, baseParams());
+
+    assert.ok(calls.length > 1, "test input should split into multiple chunks");
+    for (const call of calls) {
+      assert.ok(call.text.length <= 12, `chunk exceeded maxChars: ${call.text.length}`);
+    }
+  }
+});
+
 test("ok=false ingest responses retry the same chunk before advancing", async () => {
   const calls: Array<{ mode?: IngestMode; text: string }> = [];
   let attempt = 0;