diff --git a/Sources/SWBCore/Dependencies.swift b/Sources/SWBCore/Dependencies.swift new file mode 100644 index 00000000..12b1791e --- /dev/null +++ b/Sources/SWBCore/Dependencies.swift @@ -0,0 +1,90 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +public import SWBUtil +public import SWBMacro + +import Foundation + +// Global/target dependency settings +public struct DependencySettings: Serializable, Sendable, Encodable { + public let dependencies: [String] // lexicographically ordered and uniqued + public let verification: Bool + + public init( + dependencies: any Sequence, + verification: Bool + ) { + self.dependencies = Array(OrderedSet(dependencies).sorted()) + self.verification = verification + } + + public func serialize(to serializer: T) { + serializer.serializeAggregate(2) { + serializer.serialize(dependencies) + serializer.serialize(verification) + } + } + + public init(from deserializer: any Deserializer) throws { + try deserializer.beginAggregate(2) + self.dependencies = try deserializer.deserialize() + self.verification = try deserializer.deserialize() + } +} + +extension DependencySettings { + public init(_ scope: MacroEvaluationScope) { + let dependencies = scope.evaluate(BuiltinMacros.DEPENDENCIES) + self.init( + dependencies: dependencies, + verification: scope.evaluate(BuiltinMacros.DEPENDENCIES_VERIFICATION) + .isEnabled(onNotSet: !dependencies.isEmpty), + ) + } +} + +// Task-specific settings +public struct TaskDependencySettings: Serializable, Sendable, Encodable { + + public let traceFile: Path + public let dependencySettings: DependencySettings + + init(traceFile: Path, dependencySettings: DependencySettings) { + assert(!traceFile.isEmpty, "traceFile should never be empty") + self.traceFile = traceFile + self.dependencySettings = dependencySettings + } + + public func serialize(to serializer: T) { + serializer.serializeAggregate(2) { + serializer.serialize(traceFile) + serializer.serialize(dependencySettings) + } + } + + public init(from deserializer: any Deserializer) throws { + try deserializer.beginAggregate(2) + self.traceFile = try deserializer.deserialize() + self.dependencySettings = try deserializer.deserialize() + } + + func signatureData() -> String { + return "verify:\(dependencySettings.verification),deps:\(dependencySettings.dependencies.joined(separator: ":"))" + } + +} + +// Protocol for task payloads +public protocol TaskDependencySettingsPayload { + var taskDependencySettings: TaskDependencySettings? { get } +} diff --git a/Sources/SWBCore/PlannedTaskAction.swift b/Sources/SWBCore/PlannedTaskAction.swift index cd9d0de4..8a3ba1aa 100644 --- a/Sources/SWBCore/PlannedTaskAction.swift +++ b/Sources/SWBCore/PlannedTaskAction.swift @@ -330,6 +330,7 @@ public protocol TaskActionCreationDelegate func createValidateProductTaskAction() -> any PlannedTaskAction func createConstructStubExecutorInputFileListTaskAction() -> any PlannedTaskAction func createClangCompileTaskAction() -> any PlannedTaskAction + func createClangNonModularCompileTaskAction() -> any PlannedTaskAction func createClangScanTaskAction() -> any PlannedTaskAction func createSwiftDriverTaskAction() -> any PlannedTaskAction func createSwiftCompilationRequirementTaskAction() -> any PlannedTaskAction @@ -339,6 +340,7 @@ public protocol TaskActionCreationDelegate func createSignatureCollectionTaskAction() -> any PlannedTaskAction func createClangModuleVerifierInputGeneratorTaskAction() -> any PlannedTaskAction func createProcessSDKImportsTaskAction() -> any PlannedTaskAction + func createLdTaskAction() -> any PlannedTaskAction } extension TaskActionCreationDelegate { diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 11f2919a..29e11499 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -584,6 +584,8 @@ public final class BuiltinMacros { public static let DEFAULT_COMPILER = BuiltinMacros.declareStringMacro("DEFAULT_COMPILER") public static let DEFAULT_KEXT_INSTALL_PATH = BuiltinMacros.declareStringMacro("DEFAULT_KEXT_INSTALL_PATH") public static let DEFINES_MODULE = BuiltinMacros.declareBooleanMacro("DEFINES_MODULE") + public static let DEPENDENCIES = BuiltinMacros.declareStringListMacro("DEPENDENCIES") + public static let DEPENDENCIES_VERIFICATION = BuiltinMacros.declareEnumMacro("DEPENDENCIES_VERIFICATION") as EnumMacroDeclaration public static let DEPENDENCY_SCOPE_INCLUDES_DIRECT_DEPENDENCIES = BuiltinMacros.declareBooleanMacro("DEPENDENCY_SCOPE_INCLUDES_DIRECT_DEPENDENCIES") public static let DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER = BuiltinMacros.declareBooleanMacro("DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER") public static let __DIAGNOSE_DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER_ERROR = BuiltinMacros.declareBooleanMacro("__DIAGNOSE_DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER_ERROR") @@ -810,6 +812,7 @@ public final class BuiltinMacros { public static let LD_RUNPATH_SEARCH_PATHS = BuiltinMacros.declareStringListMacro("LD_RUNPATH_SEARCH_PATHS") public static let LD_SDK_IMPORTS_FILE = BuiltinMacros.declarePathMacro("LD_SDK_IMPORTS_FILE") public static let LD_WARN_UNUSED_DYLIBS = BuiltinMacros.declareBooleanMacro("LD_WARN_UNUSED_DYLIBS") + public static let LD_TRACE_FILE = BuiltinMacros.declarePathMacro("LD_TRACE_FILE") public static let _LD_MULTIARCH = BuiltinMacros.declareBooleanMacro("_LD_MULTIARCH") public static let _LD_MULTIARCH_PREFIX_MAP = BuiltinMacros.declareStringListMacro("_LD_MULTIARCH_PREFIX_MAP") public static let LEX = BuiltinMacros.declarePathMacro("LEX") @@ -1568,6 +1571,8 @@ public final class BuiltinMacros { DEFAULT_COMPILER, DEFAULT_KEXT_INSTALL_PATH, DEFINES_MODULE, + DEPENDENCIES, + DEPENDENCIES_VERIFICATION, DEPENDENCY_SCOPE_INCLUDES_DIRECT_DEPENDENCIES, DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER, __DIAGNOSE_DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER_ERROR, @@ -1872,6 +1877,7 @@ public final class BuiltinMacros { LD_RUNPATH_SEARCH_PATHS, LD_SDK_IMPORTS_FILE, LD_WARN_UNUSED_DYLIBS, + LD_TRACE_FILE, _LD_MULTIARCH, _LD_MULTIARCH_PREFIX_MAP, LEGACY_DEVELOPER_DIR, @@ -2826,3 +2832,22 @@ extension BuildVersion.Platform { return dtsn } } + +public enum DependenciesVerificationSetting: String, Equatable, Hashable, EnumerationMacroType { + public static let defaultValue = DependenciesVerificationSetting.notset + + case notset = "NOT_SET" + case enabled = "YES" + case disabled = "NO" + + func isEnabled(onNotSet: Bool) -> Bool { + return switch self { + case .notset: + onNotSet + case .enabled: + true + case .disabled: + false + } + } +} diff --git a/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift index 313eba1e..55a66a0d 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift @@ -415,7 +415,7 @@ struct ClangModuleVerifierPayload: ClangModuleVerifierPayloadType { } } -public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEditableTaskPayload, Encodable { +public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEditableTaskPayload, TaskDependencySettingsPayload, Encodable { let dependencyInfoEditPayload: DependencyInfoEditPayload? /// The path to the serialized diagnostic output. Every clang task must provide this path. @@ -432,7 +432,9 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd public let fileNameMapPath: Path? - fileprivate init(serializedDiagnosticsPath: Path?, indexingPayload: ClangIndexingPayload?, explicitModulesPayload: ClangExplicitModulesPayload? = nil, outputObjectFilePath: Path? = nil, fileNameMapPath: Path? = nil, developerPathString: String? = nil) { + public let taskDependencySettings: TaskDependencySettings? + + fileprivate init(serializedDiagnosticsPath: Path?, indexingPayload: ClangIndexingPayload?, explicitModulesPayload: ClangExplicitModulesPayload? = nil, outputObjectFilePath: Path? = nil, fileNameMapPath: Path? = nil, developerPathString: String? = nil, taskDependencySettings: TaskDependencySettings? = nil) { if let developerPathString, explicitModulesPayload == nil { self.dependencyInfoEditPayload = .init(removablePaths: [], removableBasenames: [], developerPath: Path(developerPathString)) } else { @@ -443,27 +445,30 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd self.explicitModulesPayload = explicitModulesPayload self.outputObjectFilePath = outputObjectFilePath self.fileNameMapPath = fileNameMapPath + self.taskDependencySettings = taskDependencySettings } public func serialize(to serializer: T) { - serializer.serializeAggregate(6) { + serializer.serializeAggregate(7) { serializer.serialize(serializedDiagnosticsPath) serializer.serialize(indexingPayload) serializer.serialize(explicitModulesPayload) serializer.serialize(outputObjectFilePath) serializer.serialize(fileNameMapPath) serializer.serialize(dependencyInfoEditPayload) + serializer.serialize(taskDependencySettings) } } public init(from deserializer: any Deserializer) throws { - try deserializer.beginAggregate(6) + try deserializer.beginAggregate(7) self.serializedDiagnosticsPath = try deserializer.deserialize() self.indexingPayload = try deserializer.deserialize() self.explicitModulesPayload = try deserializer.deserialize() self.outputObjectFilePath = try deserializer.deserialize() self.fileNameMapPath = try deserializer.deserialize() self.dependencyInfoEditPayload = try deserializer.deserialize() + self.taskDependencySettings = try deserializer.deserialize() } } @@ -1158,6 +1163,24 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible dependencyData = nil } + let taskDependencySettings = TaskDependencySettings( + traceFile: Path(outputNode.path.str + ".trace.json"), + dependencySettings: DependencySettings(cbc.scope) + ) + + if taskDependencySettings.dependencySettings.verification { + commandLine += [ + "-Xclang", + "-header-include-file", + "-Xclang", + taskDependencySettings.traceFile.str, + "-Xclang", + "-header-include-filtering=only-direct-system", + "-Xclang", + "-header-include-format=json" + ] + } + // Add the diagnostics serialization flag. We currently place the diagnostics file right next to the output object file. let diagFilePath: Path? if let serializedDiagnosticsOptions = self.serializedDiagnosticsOptions(scope: cbc.scope, outputPath: outputNode.path) { @@ -1268,7 +1291,8 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible explicitModulesPayload: explicitModulesPayload, outputObjectFilePath: shouldGenerateRemarks ? outputNode.path : nil, fileNameMapPath: verifierPayload?.fileNameMapPath, - developerPathString: recordSystemHeaderDepsOutsideSysroot ? cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR).str : nil + developerPathString: recordSystemHeaderDepsOutsideSysroot ? cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR).str : nil, + taskDependencySettings: taskDependencySettings ) var inputNodes: [any PlannedNode] = inputDeps.map { delegate.createNode($0) } @@ -1318,8 +1342,10 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible extraInputs = [] } + additionalSignatureData += taskDependencySettings.signatureData() + // Finally, create the task. - delegate.createTask(type: self, dependencyData: dependencyData, payload: payload, ruleInfo: ruleInfo, additionalSignatureData: additionalSignatureData, commandLine: commandLine, additionalOutput: additionalOutput, environment: environmentBindings, workingDirectory: compilerWorkingDirectory(cbc), inputs: inputNodes + extraInputs, outputs: [outputNode], action: action ?? delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing, additionalTaskOrderingOptions: [.compilationForIndexableSourceFile], usesExecutionInputs: usesExecutionInputs, showEnvironment: true, priority: .preferred) + delegate.createTask(type: self, dependencyData: dependencyData, payload: payload, ruleInfo: ruleInfo, additionalSignatureData: additionalSignatureData, commandLine: commandLine, additionalOutput: additionalOutput, environment: environmentBindings, workingDirectory: compilerWorkingDirectory(cbc), inputs: inputNodes + extraInputs, outputs: [outputNode], action: action ?? delegate.taskActionCreationDelegate.createClangNonModularCompileTaskAction(), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing, additionalTaskOrderingOptions: [.compilationForIndexableSourceFile], usesExecutionInputs: usesExecutionInputs, showEnvironment: true, priority: .preferred) // If the object file verifier is enabled and we are building with explicit modules, also create a job to produce adjacent objects using implicit modules, then compare the results. if cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_EXPLICIT_MODULES_OBJECT_FILE_VERIFIER) && action != nil { diff --git a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift index 8dd603b1..d1f4dc0a 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift @@ -52,7 +52,7 @@ struct LdLinkerTaskPreviewPayload: Serializable, Encodable { } } -fileprivate struct LdLinkerTaskPayload: DependencyInfoEditableTaskPayload { +fileprivate struct LdLinkerTaskPayload: DependencyInfoEditableTaskPayload, TaskDependencySettingsPayload { /// Path that points to the output linker file. let outputPath: Path @@ -68,17 +68,21 @@ fileprivate struct LdLinkerTaskPayload: DependencyInfoEditableTaskPayload { /// Path to the object file emitted during LTO, used for optimization remarks. fileprivate let objectPathLTO: Path? + public let taskDependencySettings: TaskDependencySettings? + init( outputPath: Path, dependencyInfoEditPayload: DependencyInfoEditPayload? = nil, previewPayload: LdLinkerTaskPreviewPayload? = nil, previewStyle: PreviewStyle? = nil, - objectPathLTO: Path? = nil + objectPathLTO: Path? = nil, + taskDependencySettings: TaskDependencySettings? = nil, ) { self.outputPath = outputPath self.dependencyInfoEditPayload = dependencyInfoEditPayload self.previewPayload = previewPayload self.objectPathLTO = objectPathLTO + self.taskDependencySettings = taskDependencySettings switch previewStyle { case .dynamicReplacement: self.previewStyle = .dynamicReplacement @@ -90,22 +94,24 @@ fileprivate struct LdLinkerTaskPayload: DependencyInfoEditableTaskPayload { } public func serialize(to serializer: T) { - serializer.serializeAggregate(5) { + serializer.serializeAggregate(6) { serializer.serialize(outputPath) serializer.serialize(dependencyInfoEditPayload) serializer.serialize(previewPayload) serializer.serialize(objectPathLTO) serializer.serialize(previewStyle) + serializer.serialize(taskDependencySettings) } } public init(from deserializer: any Deserializer) throws { - try deserializer.beginAggregate(5) + try deserializer.beginAggregate(6) self.outputPath = try deserializer.deserialize() self.dependencyInfoEditPayload = try deserializer.deserialize() self.previewPayload = try deserializer.deserialize() self.objectPathLTO = try deserializer.deserialize() self.previewStyle = try deserializer.deserialize() + self.taskDependencySettings = try deserializer.deserialize() } } @@ -147,6 +153,15 @@ public struct DiscoveredLdLinkerToolSpecInfo: DiscoveredCommandLineToolSpecInfo public let toolPath: Path public let toolVersion: Version? public let architectures: Set + + public func supportsTraceFile() -> Bool { + #if os(macOS) + return (try? toolVersion >= Version("1162.191")) ?? false + #else + return false + #endif + + } } /// Parses stderr output as generated by ld(1). @@ -403,6 +418,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // FIXME: Honor LD_QUITE_LINKER_ARGUMENTS_FOR_COMPILER_DRIVER == NO ? let optionContext = await discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate) + let ldOptionContext = optionContext as? DiscoveredLdLinkerToolSpecInfo // Gather additional linker arguments from the used tools. // @@ -412,7 +428,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // If we are producing an object file then the additional linker args are not added. This is because // some linkers like GNU Gold will not accept link libraries when producing object files through // partial linking and the "-r" flag. - if let linkerContext = (optionContext as? DiscoveredLdLinkerToolSpecInfo), linkerContext.linker != .ld64 && machOTypeString == "mh_object" { + if let linker = ldOptionContext?.linker, linker != .ld64 && machOTypeString == "mh_object" { continue } @@ -628,7 +644,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // Compute the inputs and outputs. var inputs: [any PlannedNode] = inputPaths.map{ delegate.createNode($0) } - await inputs.append(contentsOf: additionalInputDependencies(cbc, delegate, optionContext: discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate), lookup: lookup).map(delegate.createNode)) + inputs.append(contentsOf: additionalInputDependencies(cbc, delegate, optionContext: optionContext, lookup: lookup).map(delegate.createNode)) // Add dependencies for any arguments indicating a file path. Self.addAdditionalDependenciesFromCommandLine(cbc, commandLine, environment, &inputs, &outputs, delegate) @@ -664,12 +680,32 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec editPayload = nil } + let supportsTraceFile = !usesLDClassic && ldOptionContext?.supportsTraceFile() ?? false + let taskDependencySettings: TaskDependencySettings? = if supportsTraceFile { + TaskDependencySettings( + traceFile: cbc.scope.evaluate(BuiltinMacros.LD_TRACE_FILE), + dependencySettings: DependencySettings(cbc.scope) + ) + } else { + nil + } + + if let taskDependencySettings = taskDependencySettings { + commandLine += [ + "-Xlinker", + "-trace_file", + "-Xlinker", + taskDependencySettings.traceFile.str, + ] + } + let payload = LdLinkerTaskPayload( outputPath: cbc.output, dependencyInfoEditPayload: editPayload, previewPayload: previewPayload, previewStyle: cbc.scope.previewStyle, - objectPathLTO: shouldGenerateRemarks ? objectPathLTO : nil + objectPathLTO: shouldGenerateRemarks ? objectPathLTO : nil, + taskDependencySettings: taskDependencySettings, ) // Add dependencies on any directories in our input search paths for which the build system is creating those directories. @@ -677,7 +713,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec let otherInputs = delegate.buildDirectories.sorted().compactMap { path in ldSearchPaths.contains(path.str) ? delegate.createBuildDirectoryNode(absolutePath: path) : nil } + cbc.commandOrderingInputs // Create the task. - delegate.createTask(type: self, dependencyData: dependencyInfo, payload: payload, ruleInfo: defaultRuleInfo(cbc, delegate), commandLine: commandLine, environment: environment, workingDirectory: cbc.producer.defaultWorkingDirectory, inputs: inputs + otherInputs, outputs: outputs, action: delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing) + delegate.createTask(type: self, dependencyData: dependencyInfo, payload: payload, ruleInfo: defaultRuleInfo(cbc, delegate), additionalSignatureData: taskDependencySettings?.signatureData() ?? "", commandLine: commandLine, environment: environment, workingDirectory: cbc.producer.defaultWorkingDirectory, inputs: inputs + otherInputs, outputs: outputs, action: delegate.taskActionCreationDelegate.createLdTaskAction(), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing) } public static func addAdditionalDependenciesFromCommandLine(_ cbc: CommandBuildContext, _ commandLine: [String], _ environment: EnvironmentBindings, _ inputs: inout [any PlannedNode], _ outputs: inout [any PlannedNode], _ delegate: any TaskGenerationDelegate) { @@ -1673,7 +1709,7 @@ public final class LibtoolLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @u /// Consults the global cache of discovered info for the linker at `toolPath` and returns it, creating it if necessary. /// /// This is global and public because it is used by `SWBTaskExecution` and `CoreBasedTests`, which is the basis of many of our tests (so caching this info across tests is desirable). -public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegate: any CoreClientTargetDiagnosticProducingDelegate, at toolPath: Path) async -> (any DiscoveredCommandLineToolSpecInfo)? { +public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegate: any CoreClientTargetDiagnosticProducingDelegate, at toolPath: Path) async -> DiscoveredLdLinkerToolSpecInfo? { do { do { let commandLine = [toolPath.str, "-version_details"] diff --git a/Sources/SWBCore/Specs/CoreBuildSystem.xcspec b/Sources/SWBCore/Specs/CoreBuildSystem.xcspec index d89f2142..e242b8d1 100644 --- a/Sources/SWBCore/Specs/CoreBuildSystem.xcspec +++ b/Sources/SWBCore/Specs/CoreBuildSystem.xcspec @@ -4532,6 +4532,10 @@ When this setting is enabled: Type = Boolean; DefaultValue = "NO"; }, + { + Name = "DEPENDENCIES"; + Type = StringList; + }, ); }, ) diff --git a/Sources/SWBTaskExecution/BuildDescriptionManager.swift b/Sources/SWBTaskExecution/BuildDescriptionManager.swift index bc1da89a..ddcfa415 100644 --- a/Sources/SWBTaskExecution/BuildDescriptionManager.swift +++ b/Sources/SWBTaskExecution/BuildDescriptionManager.swift @@ -856,6 +856,10 @@ extension BuildSystemTaskPlanningDelegate: TaskActionCreationDelegate { return ClangCompileTaskAction() } + func createClangNonModularCompileTaskAction() -> any PlannedTaskAction { + return ClangNonModularCompileTaskAction() + } + func createClangScanTaskAction() -> any PlannedTaskAction { return ClangScanTaskAction() } @@ -891,6 +895,10 @@ extension BuildSystemTaskPlanningDelegate: TaskActionCreationDelegate { func createProcessSDKImportsTaskAction() -> any PlannedTaskAction { return ProcessSDKImportsTaskAction() } + + func createLdTaskAction() -> any PlannedTaskAction { + return LdTaskAction() + } } fileprivate extension BuildDescription { diff --git a/Sources/SWBTaskExecution/BuiltinTaskActionsExtension.swift b/Sources/SWBTaskExecution/BuiltinTaskActionsExtension.swift index 6474dd83..8722c6ef 100644 --- a/Sources/SWBTaskExecution/BuiltinTaskActionsExtension.swift +++ b/Sources/SWBTaskExecution/BuiltinTaskActionsExtension.swift @@ -51,7 +51,9 @@ public struct BuiltinTaskActionsExtension: TaskActionExtension { 36: ConstructStubExecutorInputFileListTaskAction.self, 37: ConcatenateTaskAction.self, 38: GenericCachingTaskAction.self, - 39: ProcessSDKImportsTaskAction.self + 39: ProcessSDKImportsTaskAction.self, + 40: LdTaskAction.self, + 41: ClangNonModularCompileTaskAction.self, ] } } diff --git a/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift index 5f26bde7..a9d4e8d5 100644 --- a/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift @@ -266,7 +266,7 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA let commandLine = command.arguments let delegate = TaskProcessDelegate(outputDelegate: outputDelegate) // The frontend invocations should be unaffected by the environment, pass an empty one. - try await spawn(commandLine: commandLine, environment: [:], workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) + try await TaskAction.spawn(commandLine: commandLine, environment: [:], workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) lastResult = delegate.commandResult if lastResult == .succeeded { @@ -287,6 +287,7 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA switch lastResult { case .some(.succeeded), .some(.skipped): + // TODO: Verify dependency trace if feature enabled continue default: // Emit the frontend command which failed, unless we have debugging enabled and printed it already @@ -431,3 +432,68 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA ) } } + +public final class ClangNonModularCompileTaskAction: TaskAction { + public override class var toolIdentifier: String { + return "ccompile-nonmodular" + } + + override public func performTaskAction( + _ task: any ExecutableTask, + dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, + executionDelegate: any TaskExecutionDelegate, + clientDelegate: any TaskExecutionClientDelegate, + outputDelegate: any TaskOutputDelegate, + ) async -> CommandResult { + return await TaskDependencyVerification.exec( + ctx: TaskExecutionContext( + task: task, + dynamicExecutionDelegate: dynamicExecutionDelegate, + executionDelegate: executionDelegate, + clientDelegate: clientDelegate, + outputDelegate: outputDelegate + ), + adapter: ClangAdapter() + ) + } + + private struct ClangAdapter: TaskDependencyVerification.Adapter { + typealias T = TraceData + + let outerTraceFileEnvVar = "CC_PRINT_HEADERS_FILE" + + func exec(ctx: TaskExecutionContext, env: [String : String]) async throws -> CommandResult { + var env = env + if let format = env.removeValue(forKey: "CC_PRINT_HEADERS_FORMAT") { + if format != "json" { + throw StubError.error("Incompatible 'CC_PRINT_HEADERS_FORMAT' environment variable value '\(format)'. Only 'json' is supported.") + } + } + + if let filtering = env.removeValue(forKey: "CC_PRINT_HEADERS_FILTERING") { + if filtering != "only-direct-system" { + throw StubError.error("Incompatible 'CC_PRINT_HEADERS_FILTERING' environment variable value '\(filtering)'. Only 'only-direct-system' is supported.") + } + } + + return try await spawn(ctx: ctx, env: env) + } + + func verify( + ctx: TaskExecutionContext, + traceData: ClangNonModularCompileTaskAction.TraceData, + dependencySettings: DependencySettings + ) throws -> Bool { + return try verifyFiles( + ctx: ctx, + files: traceData.includes ?? [], + dependencySettings: dependencySettings + ) + } + } + + private struct TraceData: Decodable { + let includes: [Path]? + } + +} diff --git a/Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift index aeb49157..2185d232 100644 --- a/Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift @@ -37,7 +37,7 @@ public final class CodeSignTaskAction: TaskAction { commandLine.insert(preEncryptHashesFlag, at: 1) } - try await spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await TaskAction.spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } catch { outputDelegate.error(error.localizedDescription) return .failed diff --git a/Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift index cac4975e..58c5bbd9 100644 --- a/Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift @@ -145,7 +145,7 @@ public final class CopyTiffTaskAction: TaskAction { let processDelegate = TaskProcessDelegate(outputDelegate: outputDelegate) do { - try await spawn(commandLine: tiffutilCommand, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await TaskAction.spawn(commandLine: tiffutilCommand, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } catch { outputDelegate.error(error.localizedDescription) return .failed diff --git a/Sources/SWBTaskExecution/TaskActions/DeferredExecutionTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/DeferredExecutionTaskAction.swift index a0307227..8fd9f7e8 100644 --- a/Sources/SWBTaskExecution/TaskActions/DeferredExecutionTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/DeferredExecutionTaskAction.swift @@ -24,7 +24,7 @@ public final class DeferredExecutionTaskAction: TaskAction { public override func performTaskAction(_ task: any ExecutableTask, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, executionDelegate: any TaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, outputDelegate: any TaskOutputDelegate) async -> CommandResult { let processDelegate = TaskProcessDelegate(outputDelegate: outputDelegate) do { - try await spawn(commandLine: Array(task.commandLineAsStrings), environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await TaskAction.spawn(commandLine: Array(task.commandLineAsStrings), environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } catch { outputDelegate.error(error.localizedDescription) return .failed @@ -50,7 +50,7 @@ fileprivate extension CommandResult { } extension TaskAction { - func spawn(commandLine: [String], environment: [String: String], workingDirectory: String, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, processDelegate: any ProcessDelegate) async throws { + static func spawn(commandLine: [String], environment: [String: String], workingDirectory: String, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, processDelegate: any ProcessDelegate) async throws { guard dynamicExecutionDelegate.allowsExternalToolExecution else { try await dynamicExecutionDelegate.spawn(commandLine: commandLine, environment: environment, workingDirectory: workingDirectory, processDelegate: processDelegate) return diff --git a/Sources/SWBTaskExecution/TaskActions/EmbedSwiftStdLibTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/EmbedSwiftStdLibTaskAction.swift index f2ded2fd..2754adf0 100644 --- a/Sources/SWBTaskExecution/TaskActions/EmbedSwiftStdLibTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/EmbedSwiftStdLibTaskAction.swift @@ -515,7 +515,7 @@ public final class EmbedSwiftStdLibTaskAction: TaskAction { let capturingDelegate = CapturingOutputDelegate(outputDelegate: outputDelegate) let processDelegate = TaskProcessDelegate(outputDelegate: capturingDelegate) - try await taskAction.spawn(commandLine: args, environment: effectiveEnvironment, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await TaskAction.spawn(commandLine: args, environment: effectiveEnvironment, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) if let error = processDelegate.executionError { throw StubError.error(error) } diff --git a/Sources/SWBTaskExecution/TaskActions/FileCopyTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/FileCopyTaskAction.swift index 94b2899d..266ebc45 100644 --- a/Sources/SWBTaskExecution/TaskActions/FileCopyTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/FileCopyTaskAction.swift @@ -132,7 +132,7 @@ public final class FileCopyTaskAction: TaskAction } for commandLine in commandLine.compileAndLink.flatMap({ [$0.compile, $0.link] }) + [commandLine.lipo] { - try await spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await TaskAction.spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } } } diff --git a/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift index bb6291c2..c333a7ea 100644 --- a/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift @@ -197,7 +197,7 @@ public final class GenericCachingTaskAction: TaskAction { } emitCacheDebuggingRemark("running sandboxed command") - try await spawn(commandLine: sandboxArgs + remappedCommandLine, environment: remappedEnvironment.bindingsDictionary, workingDirectory: cacheKey.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await TaskAction.spawn(commandLine: sandboxArgs + remappedCommandLine, environment: remappedEnvironment.bindingsDictionary, workingDirectory: cacheKey.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) if processDelegate.commandResult == .succeeded { try await withThrowingTaskGroup(of: Void.self) { group in diff --git a/Sources/SWBTaskExecution/TaskActions/LSRegisterURLTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/LSRegisterURLTaskAction.swift index 93f1d0c4..4dd6e9db 100644 --- a/Sources/SWBTaskExecution/TaskActions/LSRegisterURLTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/LSRegisterURLTaskAction.swift @@ -23,7 +23,7 @@ public final class LSRegisterURLTaskAction: TaskAction { override public func performTaskAction(_ task: any ExecutableTask, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, executionDelegate: any TaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, outputDelegate: any TaskOutputDelegate) async -> CommandResult { let processDelegate = TaskProcessDelegate(outputDelegate: outputDelegate) do { - try await spawn(commandLine: Array(task.commandLineAsStrings), environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await TaskAction.spawn(commandLine: Array(task.commandLineAsStrings), environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } catch { outputDelegate.error(error.localizedDescription) return .failed diff --git a/Sources/SWBTaskExecution/TaskActions/LdTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/LdTaskAction.swift new file mode 100644 index 00000000..d33c45c1 --- /dev/null +++ b/Sources/SWBTaskExecution/TaskActions/LdTaskAction.swift @@ -0,0 +1,89 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +public import SWBCore +import Foundation +import SWBUtil + +public final class LdTaskAction: TaskAction { + public override class var toolIdentifier: String { + return "ld" + } + + override public func performTaskAction( + _ task: any ExecutableTask, + dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, + executionDelegate: any TaskExecutionDelegate, + clientDelegate: any TaskExecutionClientDelegate, + outputDelegate: any TaskOutputDelegate, + ) async -> CommandResult { + return await TaskDependencyVerification.exec( + ctx: TaskExecutionContext( + task: task, + dynamicExecutionDelegate: dynamicExecutionDelegate, + executionDelegate: executionDelegate, + clientDelegate: clientDelegate, + outputDelegate: outputDelegate + ), + adapter: LdAdapter() + ) + } + + private struct LdAdapter: TaskDependencyVerification.Adapter { + typealias T = TraceData + + let outerTraceFileEnvVar = "LD_TRACE_FILE" + + private static let inherentDependencies = [ + "libSystem.B.tbd", + "libobjc.A.tbd", + ] + + func verify( + ctx: TaskExecutionContext, + traceData: LdTaskAction.TraceData, + dependencySettings: DependencySettings + ) throws -> Bool { + return try verifyFiles( + ctx: ctx, + files: traceData.all().filter { !LdTaskAction.LdAdapter.inherentDependencies.contains($0.basename) }, + dependencySettings: dependencySettings + ) + } + } + + private struct TraceData : Decodable { + + let dynamic: [Path]? + let weak: [Path]? + let reExports: [Path]? + let upwardDynamic: [Path]? + let delayInit: [Path]? + let archives: [Path]? + + func all() -> Set { + var all = Set() + [dynamic, weak, reExports, upwardDynamic, delayInit, archives].forEach { all.formUnion($0 ?? []) } + return all + } + + enum CodingKeys: String, CodingKey { + case reExports = "re-exports" + case dynamic = "dynamic" + case weak = "weak" + case upwardDynamic = "upward-dynamic" + case delayInit = "delay-init" + case archives = "archives" + } + + } +} diff --git a/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift index 3c308b9e..18d86994 100644 --- a/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift @@ -198,7 +198,7 @@ final public class PrecompileClangModuleTaskAction: TaskAction, BuildValueValida let delegate = TaskProcessDelegate(outputDelegate: outputDelegate) // The frontend invocations should be unaffected by the environment, pass an empty one. - try await spawn(commandLine: commandLine, environment: [:], workingDirectory: dependencyInfo.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) + try await TaskAction.spawn(commandLine: commandLine, environment: [:], workingDirectory: dependencyInfo.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) let result = delegate.commandResult ?? .failed if result == .succeeded { diff --git a/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift index db56f501..0492c84a 100644 --- a/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift @@ -520,7 +520,7 @@ public final class SwiftDriverJobTaskAction: TaskAction, BuildValueValidatingTas return .succeeded } - try await spawn(commandLine: options.commandLine, environment: environment, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) + try await TaskAction.spawn(commandLine: options.commandLine, environment: environment, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) if let error = delegate.executionError { outputDelegate.error(error) diff --git a/Sources/SWBTaskExecution/TaskActions/TaskAction.swift b/Sources/SWBTaskExecution/TaskActions/TaskAction.swift index ba5e3f1d..00c05772 100644 --- a/Sources/SWBTaskExecution/TaskActions/TaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/TaskAction.swift @@ -276,3 +276,11 @@ enum TaskActionMessage case warning(String) case note(String) } + +public struct TaskExecutionContext { + public let task: any ExecutableTask + public let dynamicExecutionDelegate: any DynamicTaskExecutionDelegate + public let executionDelegate: any TaskExecutionDelegate + public let clientDelegate: any TaskExecutionClientDelegate + public let outputDelegate: any TaskOutputDelegate +} diff --git a/Sources/SWBTaskExecution/TaskDependencyVerification.swift b/Sources/SWBTaskExecution/TaskDependencyVerification.swift new file mode 100644 index 00000000..bee304e3 --- /dev/null +++ b/Sources/SWBTaskExecution/TaskDependencyVerification.swift @@ -0,0 +1,204 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + + +public import SWBCore + +import SWBUtil +import Foundation + +// A harness for use in a task action implementation to perform trace-file-based dependency verification +public struct TaskDependencyVerification { + + public protocol Adapter where T : Decodable { + associatedtype T + + var outerTraceFileEnvVar: String? { get } + + func exec( + ctx: TaskExecutionContext, + env: [String: String] + ) async throws -> CommandResult + + func verify( + ctx: TaskExecutionContext, + traceData: T, + dependencySettings: DependencySettings + ) throws -> Bool + } + + public static func exec( + ctx: TaskExecutionContext, + adapter: any TaskDependencyVerification.Adapter + ) async -> CommandResult where T : Decodable { + do { + if let taskDependencySettings = (ctx.task.payload as? (any TaskDependencySettingsPayload))?.taskDependencySettings { + if taskDependencySettings.dependencySettings.verification { + return try await execWithDependencyVerification( + ctx: ctx, + taskDependencySettings: taskDependencySettings, + adapter: adapter + ) + } + } + + return try await adapter.exec( + ctx: ctx, + env: ctx.task.environment.bindingsDictionary + ) + } catch { + ctx.outputDelegate.error(error.localizedDescription) + return .failed + } + } + + private static func execWithDependencyVerification( + ctx: TaskExecutionContext, + taskDependencySettings: TaskDependencySettings, + adapter: any TaskDependencyVerification.Adapter + ) async throws -> CommandResult where T : Decodable { + + let traceFile = taskDependencySettings.traceFile + if ctx.executionDelegate.fs.exists(traceFile) { + try ctx.executionDelegate.fs.remove(traceFile) + } + + var env = ctx.task.environment.bindingsDictionary + let outerTraceFile = adapter.outerTraceFileEnvVar + .flatMap { env.removeValue(forKey: $0) } + .map(Path.init) + + let execResult = try await adapter.exec(ctx: ctx, env: env) + + if execResult == .succeeded { + let traceData = try readAndMaybeMergeTraceFile( + type: T.self, + fs: ctx.executionDelegate.fs, + traceFile: traceFile, + outerTraceFile: outerTraceFile, + ) + + let verified = try adapter.verify( + ctx: ctx, + traceData: traceData, + dependencySettings: taskDependencySettings.dependencySettings, + ) + + if !verified { + return .failed + } + } + + return execResult + } + + private static func readAndMaybeMergeTraceFile( + type: T.Type, + fs: any FSProxy, + traceFile: Path, + outerTraceFile: Path?, + ) throws -> T where T : Decodable { + if let outerTraceFile = outerTraceFile { + // TODO: Is this file appending concurrent-targets safe? + let traceFileContent = try fs.read(traceFile) + try fs.append(outerTraceFile, contents: traceFileContent) + return try JSONDecoder().decode(type, from: Data(traceFileContent.bytes)) + } else { + // Fast path + return try JSONDecoder().decode(type, from: fs.readMemoryMapped(traceFile)) + } + } +} + +extension TaskDependencyVerification.Adapter { + var outerTraceFileEnvVar: String? { + return nil + } + + func exec(ctx: TaskExecutionContext, env: [String: String]) async throws -> CommandResult { + return try await spawn(ctx: ctx, env: env) + } + + internal func spawn(ctx: TaskExecutionContext, env: [String: String]) async throws -> CommandResult { + let processDelegate = TaskProcessDelegate(outputDelegate: ctx.outputDelegate) + try await TaskAction.spawn( + commandLine: Array(ctx.task.commandLineAsStrings), + environment: env, + workingDirectory: ctx.task.workingDirectory.str, + dynamicExecutionDelegate: ctx.dynamicExecutionDelegate, + clientDelegate: ctx.clientDelegate, + processDelegate: processDelegate, + ) + if let error = processDelegate.executionError { + ctx.outputDelegate.error(error) + return .failed + } + + return processDelegate.commandResult ?? .failed + } + + internal func verifyFiles( + ctx: TaskExecutionContext, + files: any Sequence, + dependencySettings: DependencySettings, + ) throws -> Bool { + // Group used files by inferred logical dependency name + var used = Dictionary( + grouping: files, + by: { inferDependencyName($0) ?? "" } + ) + .mapValues { OrderedSet($0)} + + // Remove declared dependencies + dependencySettings.dependencies.forEach { used.removeValue(forKey: $0) } + + // Remove any where we could not infer the dependency + let unmapped = used.removeValue(forKey: "") ?? [] + if !unmapped.isEmpty { + ctx.outputDelegate.emitWarning("Could not infer logical dependency for: \(unmapped.map(\.str).joined(separator: ", "))") + } + + // Any left are undeclared dependencies + if !used.isEmpty { + let undeclared = used.map { + $0.key + "\n " + $0.value.map { " - " + $0.str }.joined(separator: "\n ") + } + + ctx.outputDelegate.error("Undeclared dependencies: \n " + undeclared.joined(separator: "\n ")) + + return false + } + + return true + } + + // The following is a provisional/incomplete mechanism for resolving a logical dependency from a file path. + // Ultimately, a discrete subsystem will be required that is more sophisticated than just interrogating components of the path. + // This is currently the minimal viable implementation to satisfy a functional milestone and is not intended for general use. + private func inferDependencyName(_ file: Path) -> String? { + findFrameworkName(file) ?? findLibraryName(file) + } + + private func findFrameworkName(_ file: Path) -> String? { + if file.fileExtension == "framework" { + return file.basenameWithoutSuffix + } + return file.dirname.isEmpty || file.dirname.isRoot ? nil : findFrameworkName(file.dirname) + } + + private func findLibraryName(_ file: Path) -> String? { + if file.fileExtension == "a" && file.basename.starts(with: "lib") { + return String(file.basenameWithoutSuffix.suffix(from: file.str.index(file.str.startIndex, offsetBy: 3))) + } + return nil + } +} diff --git a/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift b/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift index 516931d9..fa5359f8 100644 --- a/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift +++ b/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift @@ -204,6 +204,10 @@ extension CapturingTaskGenerationDelegate: TaskActionCreationDelegate { return ClangCompileTaskAction() } + package func createClangNonModularCompileTaskAction() -> any PlannedTaskAction { + return ClangNonModularCompileTaskAction() + } + package func createClangScanTaskAction() -> any PlannedTaskAction { return ClangScanTaskAction() } @@ -239,4 +243,8 @@ extension CapturingTaskGenerationDelegate: TaskActionCreationDelegate { package func createProcessSDKImportsTaskAction() -> any PlannedTaskAction { return ProcessSDKImportsTaskAction() } + + package func createLdTaskAction() -> any PlannedTaskAction { + return LdTaskAction() + } } diff --git a/Sources/SWBTestSupport/CoreBasedTests.swift b/Sources/SWBTestSupport/CoreBasedTests.swift index 2cfbeb6f..d098f354 100644 --- a/Sources/SWBTestSupport/CoreBasedTests.swift +++ b/Sources/SWBTestSupport/CoreBasedTests.swift @@ -132,6 +132,15 @@ extension CoreBasedTests { } } + private var ldInfo: DiscoveredLdLinkerToolSpecInfo? { + get async throws { + let (core, defaultToolchain) = try await coreAndToolchain() + let mockProducer = try MockCommandProducer(core: core, productTypeIdentifier: "com.apple.product-type.framework", platform: nil, useStandardExecutableSearchPaths: true, toolchain: defaultToolchain, fs: PseudoFS()) + let toolPath = try #require(await ldPath, "couldn't find ld in default toolchain") + return await discoveredLinkerToolsInfo(mockProducer, AlwaysDeferredCoreClientDelegate(), at: toolPath) + } + } + /// The path to the Clang compiler in the default toolchain. package var clangCompilerPath: Path { get async throws { @@ -254,17 +263,19 @@ extension CoreBasedTests { package var supportsSDKImports: Bool { get async throws { #if os(macOS) - let (core, defaultToolchain) = try await coreAndToolchain() - let toolPath = try #require(defaultToolchain.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld"), "couldn't find ld in default toolchain") - let mockProducer = try await MockCommandProducer(core: getCore(), productTypeIdentifier: "com.apple.product-type.framework", platform: nil, useStandardExecutableSearchPaths: true, toolchain: nil, fs: PseudoFS()) - let toolsInfo = await SWBCore.discoveredLinkerToolsInfo(mockProducer, AlwaysDeferredCoreClientDelegate(), at: toolPath) - return (try? toolsInfo?.toolVersion >= .init("1164")) == true + return await (try? ldInfo?.toolVersion >= .init("1164")) == true #else return false #endif } } + package var supportsLinkerTrace: Bool { + get async throws { + return try await ldInfo?.supportsTraceFile() ?? false + } + } + // Linkers package var ldPath: Path? { get async throws { diff --git a/Sources/SWBTestSupport/SkippedTestSupport.swift b/Sources/SWBTestSupport/SkippedTestSupport.swift index 08ef7190..f373379c 100644 --- a/Sources/SWBTestSupport/SkippedTestSupport.swift +++ b/Sources/SWBTestSupport/SkippedTestSupport.swift @@ -197,6 +197,12 @@ extension Trait where Self == Testing.ConditionTrait { } } + package static func requireLinkerTrace() -> Self { + enabled("Linker does not support trace") { + return try await ConditionTraitContext.shared.supportsLinkerTrace + } + } + package static func requireLocalFileSystem(_ sdks: RunDestinationInfo...) -> Self { disabled("macOS SDK is on a remote filesystem") { let core = try await ConditionTraitContext.shared.getCore() diff --git a/Sources/SWBTestSupport/TaskPlanningTestSupport.swift b/Sources/SWBTestSupport/TaskPlanningTestSupport.swift index ae67ca1c..20fa5b7d 100644 --- a/Sources/SWBTestSupport/TaskPlanningTestSupport.swift +++ b/Sources/SWBTestSupport/TaskPlanningTestSupport.swift @@ -432,6 +432,10 @@ extension TestTaskPlanningDelegate: TaskActionCreationDelegate { return ClangCompileTaskAction() } + package func createClangNonModularCompileTaskAction() -> any PlannedTaskAction { + return ClangNonModularCompileTaskAction() + } + package func createClangScanTaskAction() -> any PlannedTaskAction { return ClangScanTaskAction() } @@ -467,6 +471,10 @@ extension TestTaskPlanningDelegate: TaskActionCreationDelegate { package func createProcessSDKImportsTaskAction() -> any PlannedTaskAction { return ProcessSDKImportsTaskAction() } + + package func createLdTaskAction() -> any PlannedTaskAction { + return LdTaskAction() + } } package final class CancellingTaskPlanningDelegate: TestTaskPlanningDelegate, @unchecked Sendable { diff --git a/Sources/SWBUniversalPlatform/Plugin.swift b/Sources/SWBUniversalPlatform/Plugin.swift index b50a687c..d46463dd 100644 --- a/Sources/SWBUniversalPlatform/Plugin.swift +++ b/Sources/SWBUniversalPlatform/Plugin.swift @@ -78,6 +78,6 @@ struct UniversalPlatformTaskProducerExtension: TaskProducerExtension { struct UniversalPlatformTaskActionExtension: TaskActionExtension { var taskActionImplementations: [SWBUtil.SerializableTypeCode : any SWBUtil.PolymorphicSerializable.Type] { - [41: TestEntryPointGenerationTaskAction.self] + [42: TestEntryPointGenerationTaskAction.self] } } diff --git a/Sources/SWBUniversalPlatform/Specs/Ld.xcspec b/Sources/SWBUniversalPlatform/Specs/Ld.xcspec index d30e08ad..471faa95 100644 --- a/Sources/SWBUniversalPlatform/Specs/Ld.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/Ld.xcspec @@ -724,6 +724,12 @@ }; }, + { + Name = "LD_TRACE_FILE"; + Type = Path; + DefaultValue = "$(OBJECT_FILE_DIR_$(CURRENT_VARIANT))/$(CURRENT_ARCH)/$(PRODUCT_NAME)_trace.json"; + }, + { Name = "__CREATE_INFOPLIST_SECTION_IN_BINARY"; Type = Boolean; diff --git a/Tests/SWBBuildSystemTests/DependencyVerificationBuildOperationTests.swift b/Tests/SWBBuildSystemTests/DependencyVerificationBuildOperationTests.swift new file mode 100644 index 00000000..cf33ee08 --- /dev/null +++ b/Tests/SWBBuildSystemTests/DependencyVerificationBuildOperationTests.swift @@ -0,0 +1,102 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Testing + +import SWBCore +import SWBTestSupport +@_spi(Testing) import SWBUtil + +import SWBTaskExecution +import SWBProtocol + +@Suite(.requireSDKs(.macOS), .requireLinkerTrace()) +fileprivate struct DependencyVerificationBuildOperationTests: CoreBasedTests { + + @Test + func canVerifyDeclaredDependencies() async throws { + try await withTemporaryDirectory { tmpDirPath async throws -> Void in + let testWorkspace = TestWorkspace( + "Test", + sourceRoot: tmpDirPath.join("Test"), + projects: [ + TestProject( + "aProject", + groupTree: TestGroup( + "Sources", path: "Sources", + children: [ + TestFile("CoreFoo.m") + ]), + buildConfigurations: [ + TestBuildConfiguration( + "Debug", + buildSettings: [ + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_ENABLE_MODULES": "NO", + "GENERATE_INFOPLIST_FILE": "YES", + "DEPENDENCIES": "Foundation", + // Disable the SetOwnerAndGroup action by setting them to empty values. + "INSTALL_GROUP": "", + "INSTALL_OWNER": "", + ] + ) + ], + targets: [ + TestStandardTarget( + "CoreFoo", type: .framework, + buildPhases: [ + TestSourcesBuildPhase(["CoreFoo.m"]), + TestFrameworksBuildPhase() + ]) + ]) + ] + ) + + let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false) + let SRCROOT = testWorkspace.sourceRoot.join("aProject") + + // Write the source files. + try await tester.fs.writeFileContents(SRCROOT.join("Sources/CoreFoo.m")) { contents in + contents <<< """ + #include + #include + + void f0(void) { }; + """ + } + + func parameters(_ overrides: [String: String] = [:]) -> BuildParameters { + return BuildParameters( + action: .install, configuration: "Debug", + overrides: [ + "DSTROOT": tmpDirPath.join("dst").str + ].merging(overrides, uniquingKeysWith: { _, new in new }) + ) + } + + // Non-modular clang complains about undeclared dependency + try await tester.checkBuild(parameters: parameters(), runDestination: .macOS, persistent: true) { results in + results.checkError(.contains("Undeclared dependencies: \n Accelerate")) + } + + // Declaring dependency resolves problem + try await tester.checkBuild(parameters: parameters(["DEPENDENCIES": "Foundation Accelerate"]), runDestination: .macOS, persistent: true) { results in + results.checkNoErrors() + } + + // Linker complains about undeclared dependency + try await tester.checkBuild(parameters: parameters(["OTHER_LDFLAGS": "-framework CoreData", "DEPENDENCIES": "Foundation Accelerate"]), runDestination: .macOS, persistent: true) { results in + results.checkError(.contains("Undeclared dependencies: \n CoreData")) + } + } + } +} diff --git a/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift b/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift index 6a2eae3b..dfc1fbec 100644 --- a/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift +++ b/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift @@ -106,6 +106,10 @@ private final class CapturingTaskGenerationDelegate: TaskGenerationDelegate { } extension CapturingTaskGenerationDelegate: TaskActionCreationDelegate { + func createLdTaskAction() -> any SWBCore.PlannedTaskAction { + return LdTaskAction() + } + public func createAuxiliaryFileTaskAction(_ context: AuxiliaryFileTaskActionContext) -> any PlannedTaskAction { return AuxiliaryFileTaskAction(context) } @@ -198,6 +202,10 @@ extension CapturingTaskGenerationDelegate: TaskActionCreationDelegate { return ClangCompileTaskAction() } + public func createClangNonModularCompileTaskAction() -> any PlannedTaskAction { + return ClangNonModularCompileTaskAction() + } + public func createClangScanTaskAction() -> any PlannedTaskAction { return ClangScanTaskAction() } diff --git a/Tests/SWBCoreTests/CommandLineSpecTests.swift b/Tests/SWBCoreTests/CommandLineSpecTests.swift index 69419765..974280c8 100644 --- a/Tests/SWBCoreTests/CommandLineSpecTests.swift +++ b/Tests/SWBCoreTests/CommandLineSpecTests.swift @@ -1227,6 +1227,7 @@ import SWBMacro try table.push(core.specRegistry.internalMacroNamespace.declareStringMacro("DYNAMIC_LIBRARY_EXTENSION") as StringMacroDeclaration, literal: "dylib") try table.push(core.specRegistry.internalMacroNamespace.declareBooleanMacro("_DISCOVER_COMMAND_LINE_LINKER_INPUTS") as BooleanMacroDeclaration, literal: true) try table.push(core.specRegistry.internalMacroNamespace.declareBooleanMacro("_DISCOVER_COMMAND_LINE_LINKER_INPUTS_INCLUDE_WL") as BooleanMacroDeclaration, literal: true) + table.push(BuiltinMacros.LD_TRACE_FILE, literal: "tmp/obj/normal/x86_64/output.trace.json") let producer = try MockCommandProducer(core: core, productTypeIdentifier: "com.apple.product-type.framework", platform: "macosx") let delegate = try CapturingTaskGenerationDelegate(producer: producer, userPreferences: .defaultForTesting) @@ -1350,6 +1351,7 @@ import SWBMacro let macro = table.namespace.lookupOrDeclareMacro(type.self, name) let expr = table.namespace.parseString(value) table.push(macro, expr) + try table.push(core.specRegistry.internalMacroNamespace.declarePathMacro("LD_TRACE_FILE") as PathMacroDeclaration, literal: "tmp/obj/normal/x86_64/output.trace.json") let delegate = try CapturingTaskGenerationDelegate(producer: producer, userPreferences: .defaultForTesting) let mockScope = MacroEvaluationScope(table: table) @@ -1549,6 +1551,8 @@ import SWBMacro // We have to push this manually, since we do not have a real Setting's constructed scope. table.push(BuiltinMacros.PER_ARCH_LD, BuiltinMacros.namespace.parseString("$(LD_$(CURRENT_ARCH))")) table.push(BuiltinMacros.PER_ARCH_LDPLUSPLUS, BuiltinMacros.namespace.parseString("$(LDPLUSPLUS_$(CURRENT_ARCH))")) + table.push(BuiltinMacros.LD_TRACE_FILE, literal: "tmp/obj/normal/x86_64/output.trace.json") + let mockScope = MacroEvaluationScope(table: table) let producer = try MockCommandProducer(core: core, productTypeIdentifier: "com.apple.product-type.framework", platform: "macosx") @@ -1607,6 +1611,7 @@ import SWBMacro var table = MacroValueAssignmentTable(namespace: core.specRegistry.internalMacroNamespace) table.push(BuiltinMacros.arch, literal: "arm64e") table.push(BuiltinMacros.variant, literal: "normal") + table.push(BuiltinMacros.LD_TRACE_FILE, literal: "tmp/obj/normal/x86_64/output.trace.json") table.push(macroName, literal: value) let mockScope = MacroEvaluationScope(table: table) let producer = try MockCommandProducer(core: core, productTypeIdentifier: "com.apple.product-type.framework", platform: "macosx") diff --git a/Tests/SWBCoreTests/DependencySettingsTests.swift b/Tests/SWBCoreTests/DependencySettingsTests.swift new file mode 100644 index 00000000..e38e2c42 --- /dev/null +++ b/Tests/SWBCoreTests/DependencySettingsTests.swift @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import Testing +import SWBUtil +import SWBCore +import SWBProtocol +import SWBTestSupport +import SWBMacro + +@Suite fileprivate struct DependencySettingsTests { + + @Test + func emptyDependenciesValueDisablesVerification() throws { + #expect(!settings().verification) + #expect(settings(dependencies: ["Foo"]).verification) + #expect(settings(verification: .enabled).verification) + #expect(!settings(dependencies: ["Foo"], verification: .disabled).verification) + } + + @Test + func dependenciesAreOrderedAndUnique() throws { + #expect(Array(settings(dependencies: ["B", "A", "B", "A"]).dependencies) == ["A", "B"]) + } + + func settings(dependencies: [String]? = nil, verification: DependenciesVerificationSetting? = nil) -> DependencySettings { + var table = MacroValueAssignmentTable(namespace: BuiltinMacros.namespace) + if let verification { + table.push(BuiltinMacros.DEPENDENCIES_VERIFICATION, literal: verification) + } + if let dependencies { + table.push(BuiltinMacros.DEPENDENCIES, literal: dependencies) + } + + let scope = MacroEvaluationScope(table: table) + return DependencySettings(scope) + + } + +} diff --git a/Tests/SWBTaskConstructionTests/DependencyVerificationTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/DependencyVerificationTaskConstructionTests.swift new file mode 100644 index 00000000..caa4323a --- /dev/null +++ b/Tests/SWBTaskConstructionTests/DependencyVerificationTaskConstructionTests.swift @@ -0,0 +1,117 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Testing + +import SWBCore +import SWBTaskConstruction +import SWBTestSupport +import SWBUtil + +@Suite +fileprivate struct DependencyVerificationTaskConstructionTests: CoreBasedTests { + + let project = "TestProject" + let target = "TestTarget" + let sourceBaseName = "TestSource" + let source = "TestSource.m" + + func outputFile(_ srcroot: Path, _ filename: String) -> String { + return "\(srcroot.str)/build/\(project).build/Debug/\(target).build/Objects-normal/x86_64/\(filename)" + } + + @Test(.requireSDKs(.macOS)) + func addsTraceArgsWhenDependenciesDeclared() async throws { + try await testWith(["DEPENDENCIES": "Foo"]) { tester, srcroot in + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineContains([ + "-Xlinker", "-trace_file", + "-Xlinker", outputFile(srcroot, "\(target)_trace.json"), + ]) + } + results.checkTask(.compileC(target, fileName: source)) { task in + task.checkCommandLineContains([ + "-Xclang", "-header-include-file", + "-Xclang", outputFile(srcroot, "\(sourceBaseName).o.trace.json"), + "-Xclang", "-header-include-filtering=only-direct-system", + "-Xclang", "-header-include-format=json", + ]) + } + } + } + } + + @Test(.requireSDKs(.macOS)) + func noTraceArgsWhenDependenciesDeclared() async throws { + try await testWith([:]) { tester, srcroot in + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineDoesNotContain("-trace_file") + } + results.checkTask(.compileC(target, fileName: source)) { task in + task.checkCommandLineDoesNotContain("-header-include-file") + } + } + } + } + + @Test(.requireSDKs(.macOS)) + func canEnableVerificationOfNoDependencies() async throws { + try await testWith(["DEPENDENCIES_VERIFICATION": "YES"]) { tester, srcroot in + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineContains(["-Xlinker", "-trace_file",]) + } + } + } + } + + private func testWith( + _ buildSettings: [String: String], + _ assertions: (_ tester: TaskConstructionTester, _ srcroot: Path) async throws -> Void + ) async throws { + let testProject = TestProject( + project, + groupTree: TestGroup( + "TestGroup", + children: [ + TestFile(source) + ]), + buildConfigurations: [ + TestBuildConfiguration( + "Debug", + buildSettings: [ + "PRODUCT_NAME": "$(TARGET_NAME)", + "GENERATE_INFOPLIST_FILE": "YES", + "CLANG_ENABLE_MODULES": "YES", + ].merging(buildSettings) { _, new in new } + ) + ], + targets: [ + TestStandardTarget( + target, + type: .framework, + buildPhases: [ + TestSourcesBuildPhase([TestBuildFile(source)]) + ] + ) + ]) + + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + let SRCROOT = tester.workspace.projects[0].sourceRoot + + try await assertions(tester, SRCROOT) + } + +}