⚡ Bolt: [performance improvement] Optimize listProjects to avoid recursive file walking

.jules/bolt.md

@@ -0,0 +1,3 @@
+## 2024-03-24 - [Avoid `listFilesRecursive` for shallow searches]
+**Learning:** Finding files at known shallow depths (like `manifest.json` in project root directories) using `listFilesRecursive` on the entire disk store is extremely slow and scales poorly as project files increase (O(N) operation based on total file count rather than O(1 depth) directory count).
+**Action:** Use shallow directory reads with `readdir({ withFileTypes: true })` and read the targeted files directly and concurrently via `Promise.all` instead.

backend/src/store/localStore.ts

@@ -1,4 +1,4 @@
-import { readFile, stat, writeFile } from "node:fs/promises";
+import { readFile, stat, writeFile, readdir } from "node:fs/promises";
 import { homedir } from "node:os";
 import { dirname, join, relative } from "node:path";
 import { URL } from "node:url";
@@ -114,17 +114,32 @@ export class LocalStore {
   }
 
   async listProjects(): Promise<ProjectManifest[]> {
-    const entries = await listFilesRecursive(this.projectsDir);
-    const manifests: ProjectManifest[] = [];
-    for (const filePath of entries) {
-      if (!filePath.endsWith("manifest.json")) {
-        continue;
+    // ⚡ Bolt: Replaced O(N) recursive file walk with shallow directory read
+    // Finding known files at shallow depths (manifest.json in project roots)
+    // using recursive walk scales poorly as project files increase.
+    let entries;
+    try {
+      entries = await readdir(this.projectsDir, { withFileTypes: true });
+    } catch {
+      return [];
+    }
+
+    const manifestPromises = [];
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        const manifestPath = join(this.projectsDir, entry.name, "manifest.json");
+        manifestPromises.push(readJsonFile<ProjectManifest>(manifestPath));
       }
-      const manifest = await readJsonFile<ProjectManifest>(filePath);
+    }
+
+    const results = await Promise.all(manifestPromises);
+    const manifests: ProjectManifest[] = [];
+    for (const manifest of results) {
       if (manifest) {
         manifests.push(manifest);
       }
     }
+
     manifests.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
     return manifests;
   }