Enable Windows linker discovery

* Enable Windows linker discovery to get linker type and version
* Make Platform Registry initalization async
* Change the function signature of additionalPlatformExecutableSearchPaths to allow passing
  of a filesystem for discovery.  Plugins will need to be updated to match
  new signatures
* Add the visual studio install directory to platform search
  paths for executables.
  * Add Windows platform plugin extention to get install directory
* Add new internal build setting _LINKER_EXEC for testing purposes.
* Change the search paths of a Platform to a StackedSearchPath, as
  done with the Toolchain Registry
* Add test for discovery of windows linker

Author: kcieplak

Sources/SWBCore/Core.swift

@@ -100,9 +100,10 @@ public final class Core: Sendable {
 
             await core.initializeToolchainRegistry()
 
+            await core.initializePlatformRegistry()
+
             // Force loading SDKs.
             let sdkRegistry = core.sdkRegistry
-
             for `extension` in await pluginManager.extensions(of: SDKRegistryExtensionPoint.self) {
                 await sdkRegistry.registerSDKs(extension: `extension`, platformRegistry: core.platformRegistry)
             }
@@ -178,6 +179,7 @@ public final class Core: Sendable {
     public let connectionMode: ServiceHostConnectionMode
 
     @_spi(Testing) public init(delegate: any CoreDelegate, hostOperatingSystem: OperatingSystem, pluginManager: PluginManager, developerPath: String, inferiorProductsPath: Path?, additionalContentPaths: [Path], environment: [String:String], buildServiceModTime: Date, connectionMode: ServiceHostConnectionMode) async throws {
+
         self.delegate = delegate
         self.hostOperatingSystem = hostOperatingSystem
         self.pluginManager = pluginManager
@@ -301,31 +303,10 @@ public final class Core: Sendable {
     @_spi(Testing) public var toolchainPaths: [(Path, strict: Bool)]
 
     /// The platform registry.
-    public lazy var platformRegistry: PlatformRegistry = {
-        // FIXME: We should support building the platforms (with symlinks) locally (for `inferiorProductsPath`).
-
-        // Search the default location first (unless directed not to), then search any extra locations we've been passed.
-        var searchPaths: [Path]
-        if let onlySearchAdditionalPlatformPaths = getEnvironmentVariable("XCODE_ONLY_EXTRA_PLATFORM_FOLDERS"), onlySearchAdditionalPlatformPaths.boolValue {
-            searchPaths = []
-        }
-        else {
-            let platformsDir = self.developerPath.join("Platforms")
-            searchPaths = [platformsDir]
-            if hostOperatingSystem == .windows {
-                for dir in (try? localFS.listdir(platformsDir)) ?? [] {
-                    searchPaths.append(platformsDir.join(dir))
-                }
-            }
-        }
-        if let additionalPlatformSearchPaths = getEnvironmentVariable("XCODE_EXTRA_PLATFORM_FOLDERS") {
-            for searchPath in additionalPlatformSearchPaths.split(separator: ":") {
-                searchPaths.append(Path(searchPath))
-            }
-        }
-        searchPaths += UserDefaults.additionalPlatformSearchPaths
-        return PlatformRegistry(delegate: self.registryDelegate, searchPaths: searchPaths, hostOperatingSystem: hostOperatingSystem)
-    }()
+    let _platformRegistry: UnsafeDelayedInitializationSendableWrapper<PlatformRegistry> = .init()
+    public var platformRegistry: PlatformRegistry {
+        _platformRegistry.value
+    }
 
     @PluginExtensionSystemActor public var loadedPluginPaths: [Path] {
         pluginManager.pluginsByIdentifier.values.map(\.path)
@@ -423,6 +404,29 @@ public final class Core: Sendable {
         _specRegistry = await SpecRegistry(self.pluginManager, self.registryDelegate, searchPaths, domainInclusions, [:])
     }
 
+    private func initializePlatformRegistry() async {
+        var searchPaths: [Path]
+        let fs = localFS
+        if let onlySearchAdditionalPlatformPaths = getEnvironmentVariable("XCODE_ONLY_EXTRA_PLATFORM_FOLDERS"), onlySearchAdditionalPlatformPaths.boolValue {
+            searchPaths = []
+        } else {
+            let platformsDir = self.developerPath.join("Platforms")
+            searchPaths = [platformsDir]
+            if hostOperatingSystem == .windows {
+                for dir in (try? fs.listdir(platformsDir)) ?? [] {
+                    searchPaths.append(platformsDir.join(dir))
+                }
+            }
+        }
+        if let additionalPlatformSearchPaths = getEnvironmentVariable("XCODE_EXTRA_PLATFORM_FOLDERS") {
+            for searchPath in additionalPlatformSearchPaths.split(separator: ":") {
+                searchPaths.append(Path(searchPath))
+            }
+        }
+        searchPaths += UserDefaults.additionalPlatformSearchPaths
+        _platformRegistry.initialize(to: await PlatformRegistry(delegate: self.registryDelegate, searchPaths: searchPaths, hostOperatingSystem: hostOperatingSystem, fs: fs))
+    }
+
     /// Force all specs to be loaded.
     @_spi(Testing) public func loadAllSpecs() {
         // Load all platform domain specs first, as they provide the canonical definitions of build settings.

Sources/SWBCore/Extensions/PlatformInfoExtension.swift

@@ -30,7 +30,7 @@ public protocol PlatformInfoExtension: Sendable {
 
     func additionalKnownTestLibraryPathSuffixes() -> [Path]
 
-    func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path) -> [Path]
+    func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path, fs: any FSProxy) async -> [Path]
 
     func additionalToolchainExecutableSearchPaths(toolchainIdentifier: String, toolchainPath: Path) -> [Path]
 
@@ -54,7 +54,7 @@ extension PlatformInfoExtension {
         []
     }
 
-    public func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path) -> [Path] {
+    public func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path, fs: any FSProxy) async -> [Path] {
         []
     }
 

Sources/SWBCore/PlatformRegistry.swift

@@ -180,9 +180,9 @@ public final class Platform: Sendable {
     @_spi(Testing) public var sdks: [SDK] = []
 
     /// The list of executable search paths in the platform.
-    @_spi(Testing) public var executableSearchPaths: [Path]
+    @_spi(Testing) public var executableSearchPaths: StackedSearchPath
 
-    init(_ name: String, _ displayName: String, _ familyName: String, _ familyDisplayName: String?, _ identifier: String, _ devicePlatformName: String?, _ simulatorPlatformName: String?, _ path: Path, _ version: String?, _ productBuildVersion: String?, _ defaultSettings: [String: PropertyListItem], _ additionalInfoPlistEntries: [String: PropertyListItem], _ isDeploymentPlatform: Bool, _ specRegistryProvider: any SpecRegistryProvider, preferredArchValue: String?, executableSearchPaths: [Path]) {
+    init(_ name: String, _ displayName: String, _ familyName: String, _ familyDisplayName: String?, _ identifier: String, _ devicePlatformName: String?, _ simulatorPlatformName: String?, _ path: Path, _ version: String?, _ productBuildVersion: String?, _ defaultSettings: [String: PropertyListItem], _ additionalInfoPlistEntries: [String: PropertyListItem], _ isDeploymentPlatform: Bool, _ specRegistryProvider: any SpecRegistryProvider, preferredArchValue: String?, executableSearchPaths: [Path], fs: any FSProxy) {
         self.name = name
         self.displayName = displayName
         self.familyName = familyName
@@ -198,7 +198,7 @@ public final class Platform: Sendable {
         self.isDeploymentPlatform = isDeploymentPlatform
         self.specRegistryProvider = specRegistryProvider
         self.preferredArch = preferredArchValue
-        self.executableSearchPaths = executableSearchPaths
+        self.executableSearchPaths = StackedSearchPath(paths: executableSearchPaths, fs: fs)
         self.sdkCanonicalName = name
     }
 
@@ -319,34 +319,34 @@ public final class PlatformRegistry {
             })
     }
 
-    @_spi(Testing) public init(delegate: any PlatformRegistryDelegate, searchPaths: [Path], hostOperatingSystem: OperatingSystem) {
+    @_spi(Testing) public init(delegate: any PlatformRegistryDelegate, searchPaths: [Path], hostOperatingSystem: OperatingSystem, fs: any FSProxy) async {
         self.delegate = delegate
 
         for path in searchPaths {
-            registerPlatformsInDirectory(path)
+            await registerPlatformsInDirectory(path, fs)
         }
 
         do {
             if hostOperatingSystem.createFallbackSystemToolchain {
-                try registerFallbackSystemPlatform(operatingSystem: hostOperatingSystem)
+                try await registerFallbackSystemPlatform(operatingSystem: hostOperatingSystem, fs: fs)
             }
         } catch {
             delegate.error(error)
         }
 
-        @preconcurrency @PluginExtensionSystemActor func platformInfoExtensions() -> [any PlatformInfoExtensionPoint.ExtensionProtocol] {
-            delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self)
+        @preconcurrency @PluginExtensionSystemActor func platformInfoExtensions() async -> [any PlatformInfoExtensionPoint.ExtensionProtocol] {
+            return await delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self)
         }
 
-        for platformExtension in platformInfoExtensions() {
+        for platformExtension in await platformInfoExtensions() {
             for (path, data) in platformExtension.additionalPlatforms() {
-                registerPlatform(path, .plDict(data))
+                await registerPlatform(path, .plDict(data), fs)
             }
         }
     }
 
-    private func registerFallbackSystemPlatform(operatingSystem: OperatingSystem) throws {
-        try registerPlatform(Path("/"), .plDict(fallbackSystemPlatformSettings(operatingSystem: operatingSystem)))
+    private func registerFallbackSystemPlatform(operatingSystem: OperatingSystem, fs: any FSProxy) async throws {
+        try await registerPlatform(Path("/"), .plDict(fallbackSystemPlatformSettings(operatingSystem: operatingSystem)), fs)
     }
 
     private func fallbackSystemPlatformSettings(operatingSystem: OperatingSystem) throws -> [String: PropertyListItem] {
@@ -413,10 +413,10 @@ public final class PlatformRegistry {
     }
 
     /// Register all platforms in the given directory.
-    private func registerPlatformsInDirectory(_ path: Path) {
-        for item in (try? localFS.listdir(path))?.sorted(by: <) ?? [] {
-            let itemPath = path.join(item)
 
+    private func registerPlatformsInDirectory(_ path: Path, _ fs: any FSProxy) async {
+        for item in (try? fs.listdir(path))?.sorted(by: <) ?? [] {
+            let itemPath = path.join(item)
             // Check if this is a platform we should load
             guard itemPath.fileSuffix == ".platform" else { continue }
 
@@ -432,14 +432,15 @@ public final class PlatformRegistry {
                     continue
                 }
 
-                registerPlatform(itemPath, infoPlist)
+                await registerPlatform(itemPath, infoPlist, fs)
             } catch let err {
                 delegate.error(itemPath, "unable to load platform: 'Info.plist' was malformed: \(err)")
             }
         }
     }
 
-    private func registerPlatform(_ path: Path, _ data: PropertyListItem) {
+
+    private func registerPlatform(_ path: Path, _ data: PropertyListItem, _ fs: any FSProxy) async {
         // The data should always be a dictionary.
         guard case .plDict(var items) = data else {
             delegate.error(path, "unexpected platform data")
@@ -614,7 +615,7 @@ public final class PlatformRegistry {
             delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self)
         }
 
-        for platformExtension in platformInfoExtensions() {
+        for platformExtension in await platformInfoExtensions() {
             if let value = platformExtension.preferredArchValue(for: name) {
                 preferredArchValue = value
             }
@@ -624,8 +625,8 @@ public final class PlatformRegistry {
             path.join("usr").join("bin"),
         ]
 
-        for platformExtension in platformInfoExtensions() {
-            executableSearchPaths.append(contentsOf: platformExtension.additionalPlatformExecutableSearchPaths(platformName: name, platformPath: path))
+        for platformExtension in await platformInfoExtensions() {
+            await executableSearchPaths.append(contentsOf: platformExtension.additionalPlatformExecutableSearchPaths(platformName: name, platformPath: path, fs: localFS))
         }
 
         executableSearchPaths.append(contentsOf: [
@@ -635,7 +636,7 @@ public final class PlatformRegistry {
         ])
 
         // FIXME: Need to parse other fields. It would also be nice to diagnose unused keys like we do for Spec data (and we might want to just use the spec parser here).
-        let platform = Platform(name, displayName, familyName, familyDisplayName, identifier, devicePlatformName, simulatorPlatformName, path, version, productBuildVersion, defaultSettings, additionalInfoPlistEntries, isDeploymentPlatform, delegate, preferredArchValue: preferredArchValue, executableSearchPaths: executableSearchPaths)
+        let platform = Platform(name, displayName, familyName, familyDisplayName, identifier, devicePlatformName, simulatorPlatformName, path, version, productBuildVersion, defaultSettings, additionalInfoPlistEntries, isDeploymentPlatform, delegate, preferredArchValue: preferredArchValue, executableSearchPaths: executableSearchPaths, fs: fs)
         if let duplicatePlatform = platformsByIdentifier[identifier] {
             delegate.error(path, "platform '\(identifier)' already registered from \(duplicatePlatform.path.str)")
             return

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -811,6 +811,7 @@ public final class BuiltinMacros {
     public static let LIBTOOL_DEPENDENCY_INFO_FILE = BuiltinMacros.declarePathMacro("LIBTOOL_DEPENDENCY_INFO_FILE")
     public static let LINKER = BuiltinMacros.declareStringMacro("LINKER")
     public static let ALTERNATE_LINKER = BuiltinMacros.declareStringMacro("ALTERNATE_LINKER")
+    public static let _LINKER_EXEC = BuiltinMacros.declarePathMacro("_LINKER_EXEC")
     public static let LINK_OBJC_RUNTIME = BuiltinMacros.declareBooleanMacro("LINK_OBJC_RUNTIME")
     public static let LINK_WITH_STANDARD_LIBRARIES = BuiltinMacros.declareBooleanMacro("LINK_WITH_STANDARD_LIBRARIES")
     public static let LIPO = BuiltinMacros.declareStringMacro("LIPO")
@@ -1357,6 +1358,7 @@ public final class BuiltinMacros {
         ALL_SETTINGS,
         ALTERNATE_GROUP,
         ALTERNATE_LINKER,
+        _LINKER_EXEC,
         ALTERNATE_MODE,
         ALTERNATE_OWNER,
         ALTERNATE_PERMISSIONS_FILES,

Sources/SWBCore/Settings/Settings.swift

@@ -1015,7 +1015,7 @@ extension WorkspaceContext {
             }
 
             // Add the platform search paths.
-            for path in platform?.executableSearchPaths ?? [] {
+            for path in platform?.executableSearchPaths.paths ?? [] {
                 paths.append(path)
             }
 

Sources/SWBCore/Specs/Tools/LinkerTools.swift

@@ -401,7 +401,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
 
         // FIXME: Honor LD_QUITE_LINKER_ARGUMENTS_FOR_COMPILER_DRIVER == NO ?
 
-        let optionContext = await discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate)
+        let optionContext: (any DiscoveredCommandLineToolSpecInfo)? = await discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate)
 
         // Gather additional linker arguments from the used tools.
         //
@@ -571,7 +571,6 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
         if !usesLDClassic, supportsSDKImportsFeature, !sdkImportsInfoFile.isEmpty, cbc.scope.evaluate(BuiltinMacros.ENABLE_SDK_IMPORTS), cbc.producer.isApplePlatform {
             commandLine.insert(contentsOf: ["-Xlinker", "-sdk_imports", "-Xlinker", sdkImportsInfoFile.str, "-Xlinker", "-sdk_imports_each_object"], at: commandLine.count - 2) // This preserves the assumption that the last argument is the linker output which a few tests make.
             outputs.append(delegate.createNode(sdkImportsInfoFile))
-
             await cbc.producer.processSDKImportsSpec.createTasks(CommandBuildContext(producer: cbc.producer, scope: cbc.scope, inputs: []), delegate, ldSDKImportsPath: sdkImportsInfoFile)
         }
 
@@ -1239,11 +1238,18 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
 
     override public func discoveredCommandLineToolSpecInfo(_ producer: any CommandProducer, _ scope: MacroEvaluationScope, _ delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> (any DiscoveredCommandLineToolSpecInfo)? {
         let alternateLinker = scope.evaluate(BuiltinMacros.ALTERNATE_LINKER)
+        var linker = if alternateLinker != "" { Path(alternateLinker) } else { producer.hostOperatingSystem == .windows ? Path("link.exe") : Path("ld") }
 
-        let linkerPath = if alternateLinker != "" { Path(alternateLinker) } else { Path("ld") }
+        // ALTERNATE_LINKER is used in specs i.e. -fuseld=$ALTERNATE_LINKER, so it cannot be used to force the linker location.
+        // _LINKER_EXEC can be used to force the linker location, for usage in tests with empty pseudo filesystems.
+        let linkerExec = scope.evaluate(BuiltinMacros._LINKER_EXEC)
+        if !linkerExec.isEmpty{
+            linker = linkerExec
+        }
 
         // Create the cache key.  This is just the path to the ld linker we would invoke if we were invoking the linker directly.
-        guard let toolPath = producer.executableSearchPaths.lookup(linkerPath) else {
+        // Note: If the linker is an absolute path 'findExectable' will simply return the path to execute.
+        guard let toolPath = producer.executableSearchPaths.findExecutable(operatingSystem: producer.hostOperatingSystem, basename: linker.withoutSuffix) else {
             return nil
         }
 
@@ -1547,7 +1553,7 @@ public final class LibtoolLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @u
 public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegate: any CoreClientTargetDiagnosticProducingDelegate, at toolPath: Path) async -> (any DiscoveredCommandLineToolSpecInfo)? {
     do {
         do {
-            let commandLine = [toolPath.str, "-version_details"]
+            let commandLine = [toolPath.str,  producer.hostOperatingSystem == .windows ? "/VERSION" : "-version_details"]
             return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, commandLine, { executionResult in
                 let gnuLD = [
                     #/GNU ld version (?<version>[\d.]+)-.*/#,
@@ -1556,14 +1562,20 @@ public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegat
                 if let match = try gnuLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
                     return DiscoveredLdLinkerToolSpecInfo(linker: .gnuld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
                 }
-
                 let goLD = [
                     #/GNU gold version (?<version>[\d.]+)-.*/#,
                     #/GNU gold \(GNU Binutils.*\) (?<version>[\d.]+)/#,
                 ]
                 if let match = try goLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
                     return DiscoveredLdLinkerToolSpecInfo(linker: .gold, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
                 }
+                // link.exe has no option to simply dump the version, but if you pass an invalid argument, the program will dump a header that contains the version.
+                let linkExe = [
+                    #/Microsoft \(R\) Incremental Linker Version (?<version>[\d.]+)/#
+                ]
+                if let match = try linkExe.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
+                    return DiscoveredLdLinkerToolSpecInfo(linker: .linkExe, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
+                }
 
                 struct LDVersionDetails: Decodable {
                     let version: Version
@@ -1597,7 +1609,6 @@ public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegat
                     if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
                         return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
                     }
-
                     throw e
                 })
             })