feat(npm): Speed-up getting the remote package details

Obtaining the package details via `npm info` is a performance
bottleneck of ORT's NPM package manager. Request the package details
for all packages upfront, in parallel to reduce execution time.
Experiments on a development machine show that execution of `NpmFunTest`
now takes `1 min 13 sec` instead of `3 min 47 sec`.

Fixes: #9950.

Signed-off-by: Frank Viernau <[email protected]>

Author: fviernau

plugins/package-managers/node/src/main/kotlin/npm/Npm.kt

@@ -22,6 +22,11 @@ package org.ossreviewtoolkit.plugins.packagemanagers.node.npm
 import java.io.File
 import java.util.LinkedList
 
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.withContext
+
 import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.analyzer.PackageManagerFactory
@@ -45,6 +50,7 @@ import org.ossreviewtoolkit.utils.common.Os
 import org.ossreviewtoolkit.utils.common.ProcessCapture
 import org.ossreviewtoolkit.utils.common.collectMessages
 import org.ossreviewtoolkit.utils.common.withoutPrefix
+import org.ossreviewtoolkit.utils.ort.runBlocking
 
 import org.semver4j.RangesList
 import org.semver4j.RangesListFactory
@@ -126,6 +132,9 @@ class Npm(override val descriptor: PluginDescriptor = NpmFactory.descriptor, pri
         val project = parseProject(definitionFile, analysisRoot)
         val projectModuleInfo = listModules(workingDir, issues).undoDeduplication()
 
+        // Warm-up the cache to speed-up processing:
+        requestAllPackageDetails(projectModuleInfo)
+
         val scopeNames = Scope.entries
             .filterNot { excludes.isScopeExcluded(it.descriptor) }
             .mapTo(mutableSetOf()) { scope ->
@@ -179,13 +188,41 @@ class Npm(override val descriptor: PluginDescriptor = NpmFactory.descriptor, pri
 
         return process.extractNpmIssues()
     }
+
+    private fun requestAllPackageDetails(projectModuleInfo: ModuleInfo) {
+        runBlocking {
+            withContext(Dispatchers.IO.limitedParallelism(20)) {
+                projectModuleInfo.getAllPackageNodeModuleIds().map { packageName ->
+                    async { getRemotePackageDetails(packageName) }
+                }.awaitAll()
+            }
+        }
+    }
 }
 
 private enum class Scope(val descriptor: String) {
     DEPENDENCIES("dependencies"),
     DEV_DEPENDENCIES("devDependencies")
 }
 
+private fun ModuleInfo.getAllPackageNodeModuleIds(): Set<String> {
+    val queue = Scope.entries.flatMapTo(LinkedList()) { getScopeDependencies(it) }
+    val result = mutableSetOf<String>()
+
+    while (queue.isNotEmpty()) {
+        val info = queue.removeFirst()
+
+        @Suppress("ComplexCondition")
+        if (!info.isProject && info.isInstalled && !info.name.isNullOrBlank() && !info.version.isNullOrBlank()) {
+            result += "${info.name}@${info.version}"
+        }
+
+        Scope.entries.flatMapTo(queue) { info.getScopeDependencies(it) }
+    }
+
+    return result
+}
+
 private fun ModuleInfo.getScopeDependencies(scope: Scope) =
     when (scope) {
         Scope.DEPENDENCIES -> dependencies.values.filter { !it.dev }

plugins/package-managers/node/src/main/kotlin/npm/NpmDependencyHandler.kt

@@ -68,9 +68,9 @@ internal class NpmDependencyHandler(
         }
 }
 
-private val ModuleInfo.isInstalled: Boolean get() = path != null
+internal val ModuleInfo.isInstalled: Boolean get() = path != null
 
-private val ModuleInfo.isProject: Boolean get() = resolved == null
+internal val ModuleInfo.isProject: Boolean get() = resolved == null
 
 private val ModuleInfo.packageJsonFile: File get() =
     File(