fix: memory leak when scanning sketchbooks with large files (#2555)

Resolves #2537

Fix memory leak issue caused by inflight dependency, see isaacs/node-glob#435

Author: giacomocusinato

arduino-ide-extension/package.json

@@ -73,7 +73,7 @@
     "fast-safe-stringify": "^2.1.1",
     "filename-reserved-regex": "^2.0.0",
     "fqbn": "^1.0.5",
-    "glob": "^7.1.6",
+    "glob": "10.4.4",
     "google-protobuf": "^3.20.1",
     "hash.js": "^1.1.7",
     "is-online": "^10.0.0",
@@ -127,8 +127,8 @@
     "rimraf": "^2.6.1"
   },
   "optionalDependencies": {
-    "grpc-tools": "^1.12.4",
-    "@pingghost/protoc": "^1.0.2"
+    "@pingghost/protoc": "^1.0.2",
+    "grpc-tools": "^1.12.4"
   },
   "mocha": {
     "require": [

arduino-ide-extension/scripts/generate-protocol.js

@@ -5,7 +5,7 @@
   const path = require('node:path');
   const { mkdirSync, promises: fs, rmSync } = require('node:fs');
   const { exec } = require('./utils');
-  const glob = require('glob');
+  const { glob } = require('glob');
   const { SemVer, gte, valid: validSemVer } = require('semver');
   // Use a node-protoc fork until apple arm32 is supported
   // https://github.com/YePpHa/node-protoc/pull/10
@@ -147,16 +147,14 @@
   rmSync(out, { recursive: true, maxRetries: 5, force: true });
   mkdirSync(out, { recursive: true });
 
-  const protos = await new Promise((resolve) =>
-    glob('**/*.proto', { cwd: rpc }, (error, matches) => {
-      if (error) {
-        console.log(error.stack ?? error.message);
-        resolve([]);
-        return;
-      }
-      resolve(matches.map((filename) => path.join(rpc, filename)));
-    })
-  );
+  let protos = [];
+  try {
+    const matches = await glob('**/*.proto', { cwd: rpc });
+    protos = matches.map((filename) => path.join(rpc, filename));
+  } catch (error) {
+    console.log(error.stack ?? error.message);
+  }
+
   if (!protos || protos.length === 0) {
     console.log(`Could not find any .proto files under ${rpc}.`);
     process.exit(1);

arduino-ide-extension/src/node/sketches-service-impl.ts

@@ -8,7 +8,7 @@ import type { Mutable } from '@theia/core/lib/common/types';
 import URI from '@theia/core/lib/common/uri';
 import { FileUri } from '@theia/core/lib/node/file-uri';
 import { inject, injectable, named } from '@theia/core/shared/inversify';
-import glob from 'glob';
+import { glob } from 'glob';
 import crypto from 'node:crypto';
 import {
   CopyOptions,
@@ -853,13 +853,13 @@ export async function discoverSketches(
   container: Mutable<SketchContainer>,
   logger?: ILogger
 ): Promise<SketchContainer> {
-  const pathToAllSketchFiles = await globSketches(
+  const pathToAllSketchFiles = await glob(
     '/!(libraries|hardware)/**/*.{ino,pde}',
-    root
+    { root }
   );
   // if no match try to glob the sketchbook as a sketch folder
   if (!pathToAllSketchFiles.length) {
-    pathToAllSketchFiles.push(...(await globSketches('/*.{ino,pde}', root)));
+    pathToAllSketchFiles.push(...(await glob('/*.{ino,pde}', { root })));
   }
 
   // Sort by path length to filter out nested sketches, such as the `Nested_folder` inside the `Folder` sketch.
@@ -873,7 +873,14 @@ export async function discoverSketches(
   //    +--Nested_folder
   //       |
   //       +--Nested_folder.ino
-  pathToAllSketchFiles.sort((left, right) => left.length - right.length);
+  pathToAllSketchFiles.sort((left, right) => {
+    if (left.length === right.length) {
+      // Sort alphabetically for tests consistency
+      return left.localeCompare(right);
+    }
+    return left.length - right.length;
+  });
+
   const getOrCreateChildContainer = (
     container: SketchContainer,
     segments: string[]
@@ -974,17 +981,6 @@ export async function discoverSketches(
       uri: FileUri.create(path.dirname(pathToSketchFile)).toString(),
     });
   }
-  return prune(container);
-}
 
-async function globSketches(pattern: string, root: string): Promise<string[]> {
-  return new Promise<string[]>((resolve, reject) => {
-    glob(pattern, { root }, (error, results) => {
-      if (error) {
-        reject(error);
-      } else {
-        resolve(results);
-      }
-    });
-  });
+  return prune(container);
 }

electron-app/scripts/post-package.js

@@ -4,7 +4,7 @@
 const isCI = require('is-ci');
 const fs = require('fs');
 const path = require('path');
-const glob = require('glob');
+const { glob } = require('glob');
 const { isRelease } = require('./utils');
 const { isZip, adjustArchiveStructure } = require('./archive');