Skip to content

Enable Windows linker discovery #163

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Enable Windows linker discovery #163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f8e55fc
Enable Windows linker discovery
kcieplak Feb 14, 2025
d2f386b
* Do not invoke link.exe with invalid args, execution
kcieplak Feb 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 30 additions & 26 deletions Sources/SWBCore/Core.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,10 @@ public final class Core: Sendable {

await core.initializeToolchainRegistry()

await core.initializePlatformRegistry()

// Force loading SDKs.
let sdkRegistry = core.sdkRegistry

for `extension` in await pluginManager.extensions(of: SDKRegistryExtensionPoint.self) {
await sdkRegistry.registerSDKs(extension: `extension`, platformRegistry: core.platformRegistry)
}
Expand Down Expand Up @@ -178,6 +179,7 @@ public final class Core: Sendable {
public let connectionMode: ServiceHostConnectionMode

@_spi(Testing) public init(delegate: any CoreDelegate, hostOperatingSystem: OperatingSystem, pluginManager: PluginManager, developerPath: String, inferiorProductsPath: Path?, additionalContentPaths: [Path], environment: [String:String], buildServiceModTime: Date, connectionMode: ServiceHostConnectionMode) async throws {

self.delegate = delegate
self.hostOperatingSystem = hostOperatingSystem
self.pluginManager = pluginManager
Expand Down Expand Up @@ -301,31 +303,10 @@ public final class Core: Sendable {
@_spi(Testing) public var toolchainPaths: [(Path, strict: Bool)]

/// The platform registry.
public lazy var platformRegistry: PlatformRegistry = {
// FIXME: We should support building the platforms (with symlinks) locally (for `inferiorProductsPath`).

// Search the default location first (unless directed not to), then search any extra locations we've been passed.
var searchPaths: [Path]
if let onlySearchAdditionalPlatformPaths = getEnvironmentVariable("XCODE_ONLY_EXTRA_PLATFORM_FOLDERS"), onlySearchAdditionalPlatformPaths.boolValue {
searchPaths = []
}
else {
let platformsDir = self.developerPath.join("Platforms")
searchPaths = [platformsDir]
if hostOperatingSystem == .windows {
for dir in (try? localFS.listdir(platformsDir)) ?? [] {
searchPaths.append(platformsDir.join(dir))
}
}
}
if let additionalPlatformSearchPaths = getEnvironmentVariable("XCODE_EXTRA_PLATFORM_FOLDERS") {
for searchPath in additionalPlatformSearchPaths.split(separator: ":") {
searchPaths.append(Path(searchPath))
}
}
searchPaths += UserDefaults.additionalPlatformSearchPaths
return PlatformRegistry(delegate: self.registryDelegate, searchPaths: searchPaths, hostOperatingSystem: hostOperatingSystem)
}()
let _platformRegistry: UnsafeDelayedInitializationSendableWrapper<PlatformRegistry> = .init()
public var platformRegistry: PlatformRegistry {
_platformRegistry.value
}

@PluginExtensionSystemActor public var loadedPluginPaths: [Path] {
pluginManager.pluginsByIdentifier.values.map(\.path)
Expand Down Expand Up @@ -423,6 +404,29 @@ public final class Core: Sendable {
_specRegistry = await SpecRegistry(self.pluginManager, self.registryDelegate, searchPaths, domainInclusions, [:])
}

private func initializePlatformRegistry() async {
var searchPaths: [Path]
let fs = localFS
if let onlySearchAdditionalPlatformPaths = getEnvironmentVariable("XCODE_ONLY_EXTRA_PLATFORM_FOLDERS"), onlySearchAdditionalPlatformPaths.boolValue {
searchPaths = []
} else {
let platformsDir = self.developerPath.join("Platforms")
searchPaths = [platformsDir]
if hostOperatingSystem == .windows {
for dir in (try? fs.listdir(platformsDir)) ?? [] {
searchPaths.append(platformsDir.join(dir))
}
}
}
if let additionalPlatformSearchPaths = getEnvironmentVariable("XCODE_EXTRA_PLATFORM_FOLDERS") {
for searchPath in additionalPlatformSearchPaths.split(separator: Path.pathEnvironmentSeparator) {
searchPaths.append(Path(searchPath))
}
}
searchPaths += UserDefaults.additionalPlatformSearchPaths
_platformRegistry.initialize(to: await PlatformRegistry(delegate: self.registryDelegate, searchPaths: searchPaths, hostOperatingSystem: hostOperatingSystem, fs: fs))
}

/// Force all specs to be loaded.
@_spi(Testing) public func loadAllSpecs() {
// Load all platform domain specs first, as they provide the canonical definitions of build settings.
Expand Down
4 changes: 2 additions & 2 deletions Sources/SWBCore/Extensions/PlatformInfoExtension.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public protocol PlatformInfoExtension: Sendable {

func additionalKnownTestLibraryPathSuffixes() -> [Path]

func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path) -> [Path]
func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path, fs: any FSProxy) async -> [Path]

func additionalToolchainExecutableSearchPaths(toolchainIdentifier: String, toolchainPath: Path) -> [Path]

Expand All @@ -54,7 +54,7 @@ extension PlatformInfoExtension {
[]
}

public func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path) -> [Path] {
public func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path, fs: any FSProxy) async -> [Path] {
[]
}

Expand Down
43 changes: 22 additions & 21 deletions Sources/SWBCore/PlatformRegistry.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,9 +180,9 @@ public final class Platform: Sendable {
@_spi(Testing) public var sdks: [SDK] = []

/// The list of executable search paths in the platform.
@_spi(Testing) public var executableSearchPaths: [Path]
@_spi(Testing) public var executableSearchPaths: StackedSearchPath

init(_ name: String, _ displayName: String, _ familyName: String, _ familyDisplayName: String?, _ identifier: String, _ devicePlatformName: String?, _ simulatorPlatformName: String?, _ path: Path, _ version: String?, _ productBuildVersion: String?, _ defaultSettings: [String: PropertyListItem], _ additionalInfoPlistEntries: [String: PropertyListItem], _ isDeploymentPlatform: Bool, _ specRegistryProvider: any SpecRegistryProvider, preferredArchValue: String?, executableSearchPaths: [Path]) {
init(_ name: String, _ displayName: String, _ familyName: String, _ familyDisplayName: String?, _ identifier: String, _ devicePlatformName: String?, _ simulatorPlatformName: String?, _ path: Path, _ version: String?, _ productBuildVersion: String?, _ defaultSettings: [String: PropertyListItem], _ additionalInfoPlistEntries: [String: PropertyListItem], _ isDeploymentPlatform: Bool, _ specRegistryProvider: any SpecRegistryProvider, preferredArchValue: String?, executableSearchPaths: [Path], fs: any FSProxy) {
self.name = name
self.displayName = displayName
self.familyName = familyName
Expand All @@ -198,7 +198,7 @@ public final class Platform: Sendable {
self.isDeploymentPlatform = isDeploymentPlatform
self.specRegistryProvider = specRegistryProvider
self.preferredArch = preferredArchValue
self.executableSearchPaths = executableSearchPaths
self.executableSearchPaths = StackedSearchPath(paths: executableSearchPaths, fs: fs)
self.sdkCanonicalName = name
}

Expand Down Expand Up @@ -319,34 +319,34 @@ public final class PlatformRegistry {
})
}

@_spi(Testing) public init(delegate: any PlatformRegistryDelegate, searchPaths: [Path], hostOperatingSystem: OperatingSystem) {
@_spi(Testing) public init(delegate: any PlatformRegistryDelegate, searchPaths: [Path], hostOperatingSystem: OperatingSystem, fs: any FSProxy) async {
self.delegate = delegate

for path in searchPaths {
registerPlatformsInDirectory(path)
await registerPlatformsInDirectory(path, fs)
}

do {
if hostOperatingSystem.createFallbackSystemToolchain {
try registerFallbackSystemPlatform(operatingSystem: hostOperatingSystem)
try await registerFallbackSystemPlatform(operatingSystem: hostOperatingSystem, fs: fs)
}
} catch {
delegate.error(error)
}

@preconcurrency @PluginExtensionSystemActor func platformInfoExtensions() -> [any PlatformInfoExtensionPoint.ExtensionProtocol] {
delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self)
@preconcurrency @PluginExtensionSystemActor func platformInfoExtensions() async -> [any PlatformInfoExtensionPoint.ExtensionProtocol] {
return await delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self)
}

for platformExtension in platformInfoExtensions() {
for platformExtension in await platformInfoExtensions() {
for (path, data) in platformExtension.additionalPlatforms() {
registerPlatform(path, .plDict(data))
await registerPlatform(path, .plDict(data), fs)
}
}
}

private func registerFallbackSystemPlatform(operatingSystem: OperatingSystem) throws {
try registerPlatform(Path("/"), .plDict(fallbackSystemPlatformSettings(operatingSystem: operatingSystem)))
private func registerFallbackSystemPlatform(operatingSystem: OperatingSystem, fs: any FSProxy) async throws {
try await registerPlatform(Path("/"), .plDict(fallbackSystemPlatformSettings(operatingSystem: operatingSystem)), fs)
}

private func fallbackSystemPlatformSettings(operatingSystem: OperatingSystem) throws -> [String: PropertyListItem] {
Expand Down Expand Up @@ -413,10 +413,10 @@ public final class PlatformRegistry {
}

/// Register all platforms in the given directory.
private func registerPlatformsInDirectory(_ path: Path) {
for item in (try? localFS.listdir(path))?.sorted(by: <) ?? [] {
let itemPath = path.join(item)

private func registerPlatformsInDirectory(_ path: Path, _ fs: any FSProxy) async {
for item in (try? fs.listdir(path))?.sorted(by: <) ?? [] {
let itemPath = path.join(item)
// Check if this is a platform we should load
guard itemPath.fileSuffix == ".platform" else { continue }

Expand All @@ -432,14 +432,15 @@ public final class PlatformRegistry {
continue
}

registerPlatform(itemPath, infoPlist)
await registerPlatform(itemPath, infoPlist, fs)
} catch let err {
delegate.error(itemPath, "unable to load platform: 'Info.plist' was malformed: \(err)")
}
}
}

private func registerPlatform(_ path: Path, _ data: PropertyListItem) {

private func registerPlatform(_ path: Path, _ data: PropertyListItem, _ fs: any FSProxy) async {
// The data should always be a dictionary.
guard case .plDict(var items) = data else {
delegate.error(path, "unexpected platform data")
Expand Down Expand Up @@ -614,7 +615,7 @@ public final class PlatformRegistry {
delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self)
}

for platformExtension in platformInfoExtensions() {
for platformExtension in await platformInfoExtensions() {
if let value = platformExtension.preferredArchValue(for: name) {
preferredArchValue = value
}
Expand All @@ -624,8 +625,8 @@ public final class PlatformRegistry {
path.join("usr").join("bin"),
]

for platformExtension in platformInfoExtensions() {
executableSearchPaths.append(contentsOf: platformExtension.additionalPlatformExecutableSearchPaths(platformName: name, platformPath: path))
for platformExtension in await platformInfoExtensions() {
await executableSearchPaths.append(contentsOf: platformExtension.additionalPlatformExecutableSearchPaths(platformName: name, platformPath: path, fs: localFS))
}

executableSearchPaths.append(contentsOf: [
Expand All @@ -635,7 +636,7 @@ public final class PlatformRegistry {
])

// FIXME: Need to parse other fields. It would also be nice to diagnose unused keys like we do for Spec data (and we might want to just use the spec parser here).
let platform = Platform(name, displayName, familyName, familyDisplayName, identifier, devicePlatformName, simulatorPlatformName, path, version, productBuildVersion, defaultSettings, additionalInfoPlistEntries, isDeploymentPlatform, delegate, preferredArchValue: preferredArchValue, executableSearchPaths: executableSearchPaths)
let platform = Platform(name, displayName, familyName, familyDisplayName, identifier, devicePlatformName, simulatorPlatformName, path, version, productBuildVersion, defaultSettings, additionalInfoPlistEntries, isDeploymentPlatform, delegate, preferredArchValue: preferredArchValue, executableSearchPaths: executableSearchPaths, fs: fs)
if let duplicatePlatform = platformsByIdentifier[identifier] {
delegate.error(path, "platform '\(identifier)' already registered from \(duplicatePlatform.path.str)")
return
Expand Down
2 changes: 2 additions & 0 deletions Sources/SWBCore/Settings/BuiltinMacros.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -811,6 +811,7 @@ public final class BuiltinMacros {
public static let LIBTOOL_DEPENDENCY_INFO_FILE = BuiltinMacros.declarePathMacro("LIBTOOL_DEPENDENCY_INFO_FILE")
public static let LINKER = BuiltinMacros.declareStringMacro("LINKER")
public static let ALTERNATE_LINKER = BuiltinMacros.declareStringMacro("ALTERNATE_LINKER")
public static let _LINKER_EXEC = BuiltinMacros.declarePathMacro("_LINKER_EXEC")
public static let LINK_OBJC_RUNTIME = BuiltinMacros.declareBooleanMacro("LINK_OBJC_RUNTIME")
public static let LINK_WITH_STANDARD_LIBRARIES = BuiltinMacros.declareBooleanMacro("LINK_WITH_STANDARD_LIBRARIES")
public static let LIPO = BuiltinMacros.declareStringMacro("LIPO")
Expand Down Expand Up @@ -1357,6 +1358,7 @@ public final class BuiltinMacros {
ALL_SETTINGS,
ALTERNATE_GROUP,
ALTERNATE_LINKER,
_LINKER_EXEC,
ALTERNATE_MODE,
ALTERNATE_OWNER,
ALTERNATE_PERMISSIONS_FILES,
Expand Down
2 changes: 1 addition & 1 deletion Sources/SWBCore/Settings/Settings.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1015,7 +1015,7 @@ extension WorkspaceContext {
}

// Add the platform search paths.
for path in platform?.executableSearchPaths ?? [] {
for path in platform?.executableSearchPaths.paths ?? [] {
paths.append(path)
}

Expand Down
25 changes: 18 additions & 7 deletions Sources/SWBCore/Specs/Tools/LinkerTools.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec

// FIXME: Honor LD_QUITE_LINKER_ARGUMENTS_FOR_COMPILER_DRIVER == NO ?

let optionContext = await discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate)
let optionContext: (any DiscoveredCommandLineToolSpecInfo)? = await discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate)

// Gather additional linker arguments from the used tools.
//
Expand Down Expand Up @@ -571,7 +571,6 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
if !usesLDClassic, supportsSDKImportsFeature, !sdkImportsInfoFile.isEmpty, cbc.scope.evaluate(BuiltinMacros.ENABLE_SDK_IMPORTS), cbc.producer.isApplePlatform {
commandLine.insert(contentsOf: ["-Xlinker", "-sdk_imports", "-Xlinker", sdkImportsInfoFile.str, "-Xlinker", "-sdk_imports_each_object"], at: commandLine.count - 2) // This preserves the assumption that the last argument is the linker output which a few tests make.
outputs.append(delegate.createNode(sdkImportsInfoFile))

await cbc.producer.processSDKImportsSpec.createTasks(CommandBuildContext(producer: cbc.producer, scope: cbc.scope, inputs: []), delegate, ldSDKImportsPath: sdkImportsInfoFile)
}

Expand Down Expand Up @@ -1239,11 +1238,18 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec

override public func discoveredCommandLineToolSpecInfo(_ producer: any CommandProducer, _ scope: MacroEvaluationScope, _ delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> (any DiscoveredCommandLineToolSpecInfo)? {
let alternateLinker = scope.evaluate(BuiltinMacros.ALTERNATE_LINKER)
var linker = if alternateLinker != "" { Path(alternateLinker) } else { producer.hostOperatingSystem == .windows ? Path("link.exe") : Path("ld") }
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be ideal to use the linker driver (clang) to find the linker. That allows the customization point for the default linker (CLANG_DEFAULT_LINKER).

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mean use clang as the linker? Or something else?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, using clang as the linker driver (it knows what to do to locate and construct the invocation).

Copy link
Contributor Author

@kcieplak kcieplak Feb 18, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

clang is used to invoke the linker. The discovery of the linker via discoveredCommandLineToolSpecInfo attempts to find out details about the linker that clang would invoke. Internally depending on the linker type or the version discovered different linker options can be provided (via -Xlinker)

The ALTERNATE_LINKER build setting sets the -fuse-ld option forcing clang to use a different named linker. I will verify if on Windows clang will use link.exe by default.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Verified that indeed clang on windows by default will use link.exe.


let linkerPath = if alternateLinker != "" { Path(alternateLinker) } else { Path("ld") }
// ALTERNATE_LINKER is used in specs i.e. -fuseld=$ALTERNATE_LINKER, so it cannot be used to force the linker location.
// _LINKER_EXEC can be used to force the linker location, for usage in tests with empty pseudo filesystems.
let linkerExec = scope.evaluate(BuiltinMacros._LINKER_EXEC)
if !linkerExec.isEmpty{
linker = linkerExec
}

// Create the cache key. This is just the path to the ld linker we would invoke if we were invoking the linker directly.
guard let toolPath = producer.executableSearchPaths.lookup(linkerPath) else {
// Note: If the linker is an absolute path 'findExectable' will simply return the path to execute.
guard let toolPath = producer.executableSearchPaths.findExecutable(operatingSystem: producer.hostOperatingSystem, basename: linker.withoutSuffix) else {
return nil
}

Expand Down Expand Up @@ -1547,7 +1553,7 @@ public final class LibtoolLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @u
public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegate: any CoreClientTargetDiagnosticProducingDelegate, at toolPath: Path) async -> (any DiscoveredCommandLineToolSpecInfo)? {
do {
do {
let commandLine = [toolPath.str, "-version_details"]
let commandLine = [toolPath.str] + (producer.hostOperatingSystem == .windows ? [] : ["-version_details"])
return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, commandLine, { executionResult in
let gnuLD = [
#/GNU ld version (?<version>[\d.]+)-.*/#,
Expand All @@ -1556,14 +1562,20 @@ public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegat
if let match = try gnuLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .gnuld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

let goLD = [
#/GNU gold version (?<version>[\d.]+)-.*/#,
#/GNU gold \(GNU Binutils.*\) (?<version>[\d.]+)/#,
]
if let match = try goLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .gold, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}
// link.exe has no option to simply dump the version, running, the program will no arguments will dump a header that contains the version.
let linkExe = [
#/Microsoft \(R\) Incremental Linker Version (?<version>[\d.]+)/#
]
if let match = try linkExe.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .linkExe, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

struct LDVersionDetails: Decodable {
let version: Version
Expand Down Expand Up @@ -1597,7 +1609,6 @@ public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegat
if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

throw e
})
})
Expand Down
Loading