⚡ Bolt: [performance improvement] optimize listProjectFiles with concurrent chunked stat calls

.gitignore

@@ -125,3 +125,4 @@ walkthrough.md
 # Local planning and inspiration folders (keep tracked files as-is, ignore new ones)
 Plan/
 prism_inspo_ui/
+temp_dir

.jules/bolt.md

@@ -0,0 +1,4 @@
+
+## 2024-03-31 - [Optimize File Stats Fetching]
+**Learning:** Sequential `for...of` loops performing I/O operations (like `await stat(absolutePath)`) scale poorly in local disk stores for projects containing many files, leading to performance bottlenecks.
+**Action:** Use concurrent `Promise.all` with mapping over files to significantly improve processing speed when listing project files or performing batched independent disk operations.

backend/src/store/localStore.ts

@@ -137,19 +137,33 @@ export class LocalStore {
     const root = this.projectFilesDir(projectId);
     const files = await listFilesRecursive(root);
     const out: ProjectFileEntry[] = [];
-    for (const absolutePath of files) {
-      try {
-        const stats = await stat(absolutePath);
-        const rel = relative(root, absolutePath).replace(/\\/g, "/");
-        out.push({
-          path: rel,
-          size: stats.size,
-          modifiedAt: stats.mtime.toISOString()
-        });
-      } catch {
-        continue;
+
+    // Performance optimization: Process file stats concurrently in chunks to avoid EMFILE limits
+    const CHUNK_SIZE = 100;
+    for (let i = 0; i < files.length; i += CHUNK_SIZE) {
+      const chunk = files.slice(i, i + CHUNK_SIZE);
+      const promises = chunk.map(async (absolutePath) => {
+        try {
+          const stats = await stat(absolutePath);
+          const rel = relative(root, absolutePath).replace(/\\/g, "/");
+          return {
+            path: rel,
+            size: stats.size,
+            modifiedAt: stats.mtime.toISOString()
+          };
+        } catch {
+          return null;
+        }
+      });
+
+      const results = await Promise.all(promises);
+      for (const res of results) {
+        if (res !== null) {
+          out.push(res);
+        }
       }
     }
+
     out.sort((a, b) => a.path.localeCompare(b.path));
     return out;
   }