From b7e764a1f36b1315027b60f07148fb0310f85174 Mon Sep 17 00:00:00 2001 From: Cameron Clark Date: Sun, 17 Aug 2025 01:08:49 +0100 Subject: [PATCH] feat(configfileregistry): optimize config discovery Refactor hot paths in project setup for better performance and simplicity. - Config acquisition and parsing: Add fast-path for already-parsed configs, deduplicate concurrent parses using a wait group, use narrower locks (read locks for reads, write locks only for updates), avoid full reloads on entry creation, and support targeted reloads (file lists only). - Default project discovery: Replace recursive goroutine-based traversal with a single-threaded iterative BFS queue. Track visited paths by (path, loadKind) pair, upgrade loadKind monotonically, and exit early on a good match. This removes recursion depth issues, goroutine overload, and sync overhead, while making resolution deterministic. - Result tracking and service wiring: Use a single mutex with plain maps instead of SyncMaps and multiple locks. Ensure monotonic updates to prevent revisits with weaker loadKind. Initialize activeParsing in NewService and make cleanup consistent with BFS using plain maps under mutex. These changes eliminate duplicate parsing, cut CPU spikes, and avoid goroutine storms in large reference graphs. The design now scales linearly with project size and offers more predictable latency under load. Performance: Old design: O(E) goroutines (E = edges in tsconfig graph), with duplicate work on converging paths. Parsing was O(N * c) worst-case (N = configs, c = concurrent requests per config). New design: O(V + E) traversal (V = tsconfigs), visiting each config once per strongest loadKind. Parsing is O(N) amortized, as each config parses only once. See https://github.com/oxc-project/tsgolint/pull/99 for before/after analysis. --- internal/api/api.go | 4 +- internal/project/configfileregistry.go | 130 ++++++++++++++++----- internal/project/defaultprojectfinder.go | 138 ++++++++++++++++------- internal/project/service.go | 22 ++-- 4 files changed, 214 insertions(+), 80 deletions(-) diff --git a/internal/api/api.go b/internal/api/api.go index b1ee7c658f..0bcfb13a6e 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -65,9 +65,9 @@ func NewAPI(host APIHost, options APIOptions) *API { }, }) - api.configFileRegistry = &project.ConfigFileRegistry{ + api.configFileRegistry = project.NewConfigFileRegistry(&project.ConfigFileRegistry{ Host: api, - } + }) return api } diff --git a/internal/project/configfileregistry.go b/internal/project/configfileregistry.go index fc4600e4af..2eeecc71d9 100644 --- a/internal/project/configfileregistry.go +++ b/internal/project/configfileregistry.go @@ -28,12 +28,24 @@ type ExtendedConfigFileEntry struct { configFiles collections.Set[tspath.Path] } +type parseStatus struct { + wg *sync.WaitGroup +} + type ConfigFileRegistry struct { Host ProjectHost defaultProjectFinder *defaultProjectFinder ConfigFiles collections.SyncMap[tspath.Path, *ConfigFileEntry] ExtendedConfigCache collections.SyncMap[tspath.Path, *tsoptions.ExtendedConfigCacheEntry] ExtendedConfigsUsedBy collections.SyncMap[tspath.Path, *ExtendedConfigFileEntry] + + parseMu sync.Mutex + activeParsing map[tspath.Path]*parseStatus +} + +func NewConfigFileRegistry(registry *ConfigFileRegistry) *ConfigFileRegistry { + registry.activeParsing = make(map[tspath.Path]*parseStatus) + return registry } func (e *ConfigFileEntry) SetPendingReload(level PendingReload) bool { @@ -74,41 +86,99 @@ func (c *ConfigFileRegistry) releaseConfig(path tspath.Path, project *Project) { } func (c *ConfigFileRegistry) acquireConfig(fileName string, path tspath.Path, project *Project, info *ScriptInfo) *tsoptions.ParsedCommandLine { - entry, ok := c.ConfigFiles.Load(path) - if !ok { - // Create parsed command line + if entry, ok := c.ConfigFiles.Load(path); ok { + entry.mu.RLock() + needsReload := entry.pendingReload != PendingReloadNone + commandLine := entry.commandLine + entry.mu.RUnlock() + + if !needsReload && commandLine != nil { + entry.mu.Lock() + if project != nil { + entry.projects.Add(project) + } else if info != nil { + entry.infos.Add(info) + } + entry.mu.Unlock() + return commandLine + } + } + + c.parseMu.Lock() + if status, ok := c.activeParsing[path]; ok { + c.parseMu.Unlock() + status.wg.Wait() + + if entry, ok := c.ConfigFiles.Load(path); ok { + entry.mu.Lock() + defer entry.mu.Unlock() + if project != nil { + entry.projects.Add(project) + } else if info != nil { + entry.infos.Add(info) + } + return entry.commandLine + } + } + + status := &parseStatus{wg: &sync.WaitGroup{}} + status.wg.Add(1) + c.activeParsing[path] = status + c.parseMu.Unlock() + + defer func() { + status.wg.Done() + c.parseMu.Lock() + delete(c.activeParsing, path) + c.parseMu.Unlock() + }() + + entry, loaded := c.ConfigFiles.Load(path) + if !loaded { config, _ := tsoptions.GetParsedCommandLineOfConfigFilePath(fileName, path, nil, c.Host, &c.ExtendedConfigCache) var rootFilesWatch *watchedFiles[[]string] client := c.Host.Client() if c.Host.IsWatchEnabled() && client != nil { - rootFilesWatch = newWatchedFiles(&configFileWatchHost{fileName: fileName, host: c.Host}, lsproto.WatchKindChange|lsproto.WatchKindCreate|lsproto.WatchKindDelete, core.Identity, "root files") + rootFilesWatch = newWatchedFiles(&configFileWatchHost{fileName: fileName, host: c.Host}, + lsproto.WatchKindChange|lsproto.WatchKindCreate|lsproto.WatchKindDelete, + core.Identity, "root files") } - entry, _ = c.ConfigFiles.LoadOrStore(path, &ConfigFileEntry{ + + newEntry := &ConfigFileEntry{ commandLine: config, - pendingReload: PendingReloadFull, + pendingReload: PendingReloadNone, // Already parsed, no reload needed rootFilesWatch: rootFilesWatch, - }) + } + + entry, loaded = c.ConfigFiles.LoadOrStore(path, newEntry) + if !loaded { + c.updateRootFilesWatch(fileName, entry) + c.updateExtendedConfigsUsedBy(path, entry, nil) + } } + entry.mu.Lock() defer entry.mu.Unlock() + if project != nil { entry.projects.Add(project) } else if info != nil { entry.infos.Add(info) } - if entry.pendingReload == PendingReloadNone { - return entry.commandLine - } - switch entry.pendingReload { - case PendingReloadFileNames: - entry.commandLine = entry.commandLine.ReloadFileNamesOfParsedCommandLine(c.Host.FS()) - case PendingReloadFull: - oldCommandLine := entry.commandLine - entry.commandLine, _ = tsoptions.GetParsedCommandLineOfConfigFilePath(fileName, path, nil, c.Host, &c.ExtendedConfigCache) - c.updateExtendedConfigsUsedBy(path, entry, oldCommandLine) - c.updateRootFilesWatch(fileName, entry) + + if entry.pendingReload != PendingReloadNone { + switch entry.pendingReload { + case PendingReloadFileNames: + entry.commandLine = entry.commandLine.ReloadFileNamesOfParsedCommandLine(c.Host.FS()) + case PendingReloadFull: + oldCommandLine := entry.commandLine + entry.commandLine, _ = tsoptions.GetParsedCommandLineOfConfigFilePath(fileName, path, nil, c.Host, &c.ExtendedConfigCache) + c.updateExtendedConfigsUsedBy(path, entry, oldCommandLine) + c.updateRootFilesWatch(fileName, entry) + } + entry.pendingReload = PendingReloadNone } - entry.pendingReload = PendingReloadNone + return entry.commandLine } @@ -164,17 +234,19 @@ func (c *ConfigFileRegistry) updateExtendedConfigsUsedBy(path tspath.Path, entry extendedEntry.configFiles.Add(path) extendedEntry.mu.Unlock() } - for _, extendedConfig := range oldCommandLine.ExtendedSourceFiles() { - extendedPath := tspath.ToPath(extendedConfig, c.Host.GetCurrentDirectory(), c.Host.FS().UseCaseSensitiveFileNames()) - if !slices.Contains(newConfigs, extendedPath) { - extendedEntry, _ := c.ExtendedConfigsUsedBy.Load(extendedPath) - extendedEntry.mu.Lock() - extendedEntry.configFiles.Delete(path) - if extendedEntry.configFiles.Len() == 0 { - c.ExtendedConfigsUsedBy.Delete(extendedPath) - c.ExtendedConfigCache.Delete(extendedPath) + if oldCommandLine != nil { + for _, extendedConfig := range oldCommandLine.ExtendedSourceFiles() { + extendedPath := tspath.ToPath(extendedConfig, c.Host.GetCurrentDirectory(), c.Host.FS().UseCaseSensitiveFileNames()) + if !slices.Contains(newConfigs, extendedPath) { + extendedEntry, _ := c.ExtendedConfigsUsedBy.Load(extendedPath) + extendedEntry.mu.Lock() + extendedEntry.configFiles.Delete(path) + if extendedEntry.configFiles.Len() == 0 { + c.ExtendedConfigsUsedBy.Delete(extendedPath) + c.ExtendedConfigCache.Delete(extendedPath) + } + extendedEntry.mu.Unlock() } - extendedEntry.mu.Unlock() } } } diff --git a/internal/project/defaultprojectfinder.go b/internal/project/defaultprojectfinder.go index 05688bd9cd..eb92b8595b 100644 --- a/internal/project/defaultprojectfinder.go +++ b/internal/project/defaultprojectfinder.go @@ -5,7 +5,6 @@ import ( "strings" "sync" - "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/tsoptions" "github.com/microsoft/typescript-go/internal/tspath" @@ -190,33 +189,81 @@ func (f *defaultProjectFinder) tryFindDefaultConfiguredProjectFromReferences( if len(config.ProjectReferences()) == 0 { return false } - wg := core.NewWorkGroup(false) - f.tryFindDefaultConfiguredProjectFromReferencesWorker(info, config, loadKind, result, wg) - wg.RunAndWait() - return result.isDone() -} -func (f *defaultProjectFinder) tryFindDefaultConfiguredProjectFromReferencesWorker( - info *ScriptInfo, - config *tsoptions.ParsedCommandLine, - loadKind projectLoadKind, - result *openScriptInfoProjectResult, - wg core.WorkGroup, -) { + type configItem struct { + fileName string + path tspath.Path + config *tsoptions.ParsedCommandLine + loadKind projectLoadKind + } + + type visitKey struct { + path tspath.Path + loadKind projectLoadKind + } + + var queue []configItem + visited := make(map[visitKey]bool) + + initialLoadKind := loadKind if config.CompilerOptions().DisableReferencedProjectLoad.IsTrue() { - loadKind = projectLoadKindFind + initialLoadKind = projectLoadKindFind } + for _, childConfigFileName := range config.ResolvedProjectReferencePaths() { - wg.Queue(func() { + childConfigFilePath := f.service.toPath(childConfigFileName) + key := visitKey{path: childConfigFilePath, loadKind: initialLoadKind} + if !visited[key] { + visited[key] = true + childConfig := f.findOrAcquireConfig(info, childConfigFileName, childConfigFilePath, initialLoadKind) + if childConfig != nil { + queue = append(queue, configItem{ + fileName: childConfigFileName, + path: childConfigFilePath, + config: childConfig, + loadKind: initialLoadKind, + }) + } + } + } + + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + + if f.isDefaultConfigForScriptInfo(info, current.fileName, current.path, current.config, current.loadKind, result) { + return true + } + + if result.isDone() { + return true + } + + childLoadKind := current.loadKind + if current.config.CompilerOptions().DisableReferencedProjectLoad.IsTrue() { + childLoadKind = projectLoadKindFind + } + + for _, childConfigFileName := range current.config.ResolvedProjectReferencePaths() { childConfigFilePath := f.service.toPath(childConfigFileName) - childConfig := f.findOrAcquireConfig(info, childConfigFileName, childConfigFilePath, loadKind) - if childConfig == nil || f.isDefaultConfigForScriptInfo(info, childConfigFileName, childConfigFilePath, childConfig, loadKind, result) { - return + key := visitKey{path: childConfigFilePath, loadKind: childLoadKind} + + if !visited[key] { + visited[key] = true + childConfig := f.findOrAcquireConfig(info, childConfigFileName, childConfigFilePath, childLoadKind) + if childConfig != nil { + queue = append(queue, configItem{ + fileName: childConfigFileName, + path: childConfigFilePath, + config: childConfig, + loadKind: childLoadKind, + }) + } } - // Search in references if we cant find default project in current config - f.tryFindDefaultConfiguredProjectFromReferencesWorker(info, childConfig, loadKind, result, wg) - }) + } } + + return result.isDone() } func (f *defaultProjectFinder) tryFindDefaultConfiguredProjectFromAncestor( @@ -318,57 +365,70 @@ func (f *defaultProjectFinder) findDefaultConfiguredProject(scriptInfo *ScriptIn } type openScriptInfoProjectResult struct { - projectMu sync.RWMutex - project *Project - fallbackDefaultMu sync.RWMutex - fallbackDefault *Project // use this if we cant find actual project - seenProjects collections.SyncMap[*Project, projectLoadKind] - seenConfigs collections.SyncMap[tspath.Path, projectLoadKind] + mu sync.Mutex + project *Project + fallbackDefault *Project // use this if we cant find actual project + seenProjects map[*Project]projectLoadKind + seenConfigs map[tspath.Path]projectLoadKind } func (r *openScriptInfoProjectResult) addSeenProject(project *Project, loadKind projectLoadKind) bool { - if kind, loaded := r.seenProjects.LoadOrStore(project, loadKind); loaded { + r.mu.Lock() + defer r.mu.Unlock() + + if r.seenProjects == nil { + r.seenProjects = make(map[*Project]projectLoadKind) + } + + if kind, exists := r.seenProjects[project]; exists { if kind >= loadKind { return false } - r.seenProjects.Store(project, loadKind) } + r.seenProjects[project] = loadKind return true } func (r *openScriptInfoProjectResult) addSeenConfig(configPath tspath.Path, loadKind projectLoadKind) bool { - if kind, loaded := r.seenConfigs.LoadOrStore(configPath, loadKind); loaded { + r.mu.Lock() + defer r.mu.Unlock() + + if r.seenConfigs == nil { + r.seenConfigs = make(map[tspath.Path]projectLoadKind) + } + + if kind, exists := r.seenConfigs[configPath]; exists { if kind >= loadKind { return false } - r.seenConfigs.Store(configPath, loadKind) } + r.seenConfigs[configPath] = loadKind return true } func (r *openScriptInfoProjectResult) isDone() bool { - r.projectMu.RLock() - defer r.projectMu.RUnlock() + r.mu.Lock() + defer r.mu.Unlock() return r.project != nil } func (r *openScriptInfoProjectResult) setProject(project *Project) { - r.projectMu.Lock() - defer r.projectMu.Unlock() + r.mu.Lock() + defer r.mu.Unlock() if r.project == nil { r.project = project } } func (r *openScriptInfoProjectResult) hasFallbackDefault() bool { - r.fallbackDefaultMu.RLock() - defer r.fallbackDefaultMu.RUnlock() + r.mu.Lock() + defer r.mu.Unlock() return r.fallbackDefault != nil } func (r *openScriptInfoProjectResult) setFallbackDefault(project *Project) { - r.fallbackDefaultMu.Lock() - defer r.fallbackDefaultMu.Unlock() + r.mu.Lock() + defer r.mu.Unlock() if r.fallbackDefault == nil { r.fallbackDefault = project } diff --git a/internal/project/service.go b/internal/project/service.go index 9fbf1c5ed4..4469dd2baf 100644 --- a/internal/project/service.go +++ b/internal/project/service.go @@ -90,10 +90,10 @@ func NewService(host ServiceHost, options ServiceOptions) *Service { configFileForOpenFiles: make(map[tspath.Path]string), configFilesAncestorForOpenFiles: make(map[tspath.Path]map[string]string), } - service.configFileRegistry = &ConfigFileRegistry{ + service.configFileRegistry = NewConfigFileRegistry(&ConfigFileRegistry{ Host: service, defaultProjectFinder: service.defaultProjectFinder, - } + }) service.converters = ls.NewConverters(options.PositionEncoding, func(fileName string) *ls.LineMap { return service.documentStore.GetScriptInfoByPath(service.toPath(fileName)).LineMap() }) @@ -497,14 +497,16 @@ func (s *Service) cleanupConfiguredProjects(openInfo *ScriptInfo, retainedByOpen if r == nil { return } - r.seenProjects.Range(func(project *Project, _ projectLoadKind) bool { - delete(toRemoveProjects, project.configFilePath) - return true - }) - r.seenConfigs.Range(func(config tspath.Path, _ projectLoadKind) bool { - delete(toRemoveConfigs, config) - return true - }) + { + r.mu.Lock() + defer r.mu.Unlock() + for project := range r.seenProjects { + delete(toRemoveProjects, project.configFilePath) + } + for config := range r.seenConfigs { + delete(toRemoveConfigs, config) + } + } // // Keep original projects used // markOriginalProjectsAsUsed(project); // // Keep all the references alive