diff --git a/.gitignore b/.gitignore index 6ff2ddb..51fa906 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ walkthrough.md # Local planning and inspiration folders (keep tracked files as-is, ignore new ones) Plan/ prism_inspo_ui/ +temp_dir diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..18462d3 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,4 @@ + +## 2024-03-31 - [Optimize File Stats Fetching] +**Learning:** Sequential `for...of` loops performing I/O operations (like `await stat(absolutePath)`) scale poorly in local disk stores for projects containing many files, leading to performance bottlenecks. +**Action:** Use concurrent `Promise.all` with mapping over files to significantly improve processing speed when listing project files or performing batched independent disk operations. diff --git a/backend/src/store/localStore.ts b/backend/src/store/localStore.ts index 6aa075d..9f7d50f 100644 --- a/backend/src/store/localStore.ts +++ b/backend/src/store/localStore.ts @@ -137,19 +137,33 @@ export class LocalStore { const root = this.projectFilesDir(projectId); const files = await listFilesRecursive(root); const out: ProjectFileEntry[] = []; - for (const absolutePath of files) { - try { - const stats = await stat(absolutePath); - const rel = relative(root, absolutePath).replace(/\\/g, "/"); - out.push({ - path: rel, - size: stats.size, - modifiedAt: stats.mtime.toISOString() - }); - } catch { - continue; + + // Performance optimization: Process file stats concurrently in chunks to avoid EMFILE limits + const CHUNK_SIZE = 100; + for (let i = 0; i < files.length; i += CHUNK_SIZE) { + const chunk = files.slice(i, i + CHUNK_SIZE); + const promises = chunk.map(async (absolutePath) => { + try { + const stats = await stat(absolutePath); + const rel = relative(root, absolutePath).replace(/\\/g, "/"); + return { + path: rel, + size: stats.size, + modifiedAt: stats.mtime.toISOString() + }; + } catch { + return null; + } + }); + + const results = await Promise.all(promises); + for (const res of results) { + if (res !== null) { + out.push(res); + } } } + out.sort((a, b) => a.path.localeCompare(b.path)); return out; }