diff --git a/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift index 68d76eaa7..e30ba684a 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift @@ -910,7 +910,6 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible func cachingBuildEnabled( _ cbc: CommandBuildContext, - language: GCCCompatibleLanguageDialect, clangInfo: DiscoveredClangToolSpecInfo? ) -> Bool { // Disabling compilation caching for index build, for now. @@ -918,28 +917,44 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible return false } - let enabledCppModules: Bool = { - guard language.isPlusPlus else { - return false - } - // When response file is used the flag is in the response file, not in the commandLine array, - // so check the build setting. - if cbc.scope.evaluate(BuiltinMacros.OTHER_CPLUSPLUSFLAGS).contains("-fcxx-modules") { - return true - } - return false - }() - - guard !enabledCppModules else { + // If this project is on the EBM block list, the project cannot use caching. + if clangInfo?.isClangExplicitModulesBlocked(cbc.producer, cbc.scope) == true { return false } let buildSettingEnabled = cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_COMPILE_CACHE) - // If this project is on the blocklist, override the blocklist default enable for it. + // If this project is on the caching blocklist, override the blocklist default enable for it. return clangInfo?.isCachingBlocked(cbc.producer, cbc.scope) == true ? false : buildSettingEnabled } + private func lastCxxModulesFlag(_ flags: [String]) -> Bool? { + guard let flag = flags.last(where: { $0 == "-fcxx-modules" || $0 == "-fno-cxx-modules" }) else { + return nil + } + return flag == "-fcxx-modules" + } + + /// Whether clang builds C++ header modules for this TU. + /// Build settings are evaluated rather than scanning `commandLine`: with `CLANG_USE_RESPONSE_FILE` + /// enabled — the default — the flags are written to a response file and are absent from that array. + /// `OTHER_CFLAGS` is not checked because for C++/ObjC++ inputs it is replaced by + /// `OTHER_CPLUSPLUSFLAGS`, which defaults to `$(OTHER_CFLAGS)` anyway. + func cxxHeaderModulesEnabled(_ cbc: CommandBuildContext, _ input: FileToBuild) -> Bool { + guard cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_MODULES) else { + return false + } + + if let perFileArgs = input.additionalArgs, + let decision = lastCxxModulesFlag(cbc.scope.evaluate(perFileArgs)) { + return decision + } + if let decision = lastCxxModulesFlag(cbc.scope.evaluate(BuiltinMacros.OTHER_CPLUSPLUSFLAGS)) { + return decision + } + return !cbc.scope.evaluate(BuiltinMacros.CLANG_DISABLE_CXX_MODULES) + } + private func createExplicitModulesActionAndPayload(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, _ compilerLauncher: Path?, _ input: FileToBuild, _ language: GCCCompatibleLanguageDialect?, commandLine: [String], scanningOutputPath: Path, isForPCHTask: Bool, clangInfo: DiscoveredClangToolSpecInfo?) -> (action: (any PlannedTaskAction)?, usesExecutionInputs: Bool, payload: ClangExplicitModulesPayload?, signatureData: String?) { guard let language else { // Unknown language. @@ -951,21 +966,30 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible return (nil, false, nil, nil) } - let cachedBuild = cachingBuildEnabled(cbc, language: language, clangInfo: clangInfo) + // The dependency scanner should only be used for C-family languages. + let isCFamily = GCCCompatibleLanguageDialect.allCLanguages.contains(language) + + // Whether clang modules are enabled for this translation unit. C++/ObjC++ additionally + // require the project to opt in, since `-fmodules` alone does not enable C++ header modules. + let modulesEnabled = isCFamily + && cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_MODULES) + && (!language.isPlusPlus || cxxHeaderModulesEnabled(cbc, input)) + + // Whether explicitly built modules are in effect. A project on the blocklist is known to fail + // under them. For such a project, EBM is off and the build falls back to implicit modules. let explicitModules = cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_MODULES) && (cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_EXPLICIT_MODULES) || cbc.scope.evaluate(BuiltinMacros._EXPERIMENTAL_CLANG_EXPLICIT_MODULES)) + && clangInfo?.isClangExplicitModulesBlocked(cbc.producer, cbc.scope) != true - let explicitModulesLanguages: Set = [ - .c, .objectiveC - ] - let supportedLanguages = cachedBuild ? GCCCompatibleLanguageDialect.allCLanguages : explicitModulesLanguages + // Whether compile caching is on. Caching requires EBMs. + let cachedBuild = cachingBuildEnabled(cbc, clangInfo: clangInfo) // Only enable dep scanner if requested by the user and if the language supports it. - EXPLICIT_MODULES: if cachedBuild || explicitModules, supportedLanguages.contains(language) { + EXPLICIT_MODULES: if isCFamily, cachedBuild || (explicitModules && modulesEnabled) { let usesCompilerLauncher = compilerLauncher != nil - if !explicitModules && explicitModulesLanguages.contains(language) && cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_MODULES) { + if cachedBuild && modulesEnabled && !explicitModules { delegate.warning("Compile caching is not supported with implicit modules; enable CLANG_ENABLE_EXPLICIT_MODULES") break EXPLICIT_MODULES } diff --git a/Sources/SWBCore/ToolInfo/ClangToolInfo.swift b/Sources/SWBCore/ToolInfo/ClangToolInfo.swift index c614d1cda..742e7757b 100644 --- a/Sources/SWBCore/ToolInfo/ClangToolInfo.swift +++ b/Sources/SWBCore/ToolInfo/ClangToolInfo.swift @@ -37,6 +37,17 @@ public struct ClangBlocklists : Sendable { var builtinModuleVerify: BuiltinModuleVerifierInfo? = nil + public struct ClangExplicitModulesInfo : ProjectFailuresBlockList, Codable, Sendable { + /// A blocklist of project names that do not support clang explicitly built modules. + /// These project also lose caching. + let KnownFailures: [String] + enum CodingKeys: String, CodingKey { + case KnownFailures + } + } + + var clangExplicitModules: ClangExplicitModulesInfo? = nil + /// Helper method for determining if a given functionality is blocklisted for the active scope. func isBlocked(_ producer: any CommandProducer, _ scope: MacroEvaluationScope, info: BlockListT?) -> Bool { guard let blocklistInfo = info else { return false } @@ -101,6 +112,10 @@ public struct DiscoveredClangToolSpecInfo: DiscoveredCommandLineToolSpecInfo { public func isBuiltinModuleVerifyBlocked(_ producer: any CommandProducer, _ scope: MacroEvaluationScope) -> Bool { return blocklists.isBlocked(producer, scope, info: blocklists.builtinModuleVerify) } + + public func isClangExplicitModulesBlocked(_ producer: any CommandProducer, _ scope: MacroEvaluationScope) -> Bool { + return blocklists.isBlocked(producer, scope, info: blocklists.clangExplicitModules) + } } private let clangVersionRe = RegEx(patternLiteral: #""(?[0-9]+(?:\.[0-9]+){0,}) \(clang-(?[0-9]+(?:\.[0-9]+){0,})\)(?: ((\[.+\])|(\(.+\)))+)?""#) @@ -199,7 +214,7 @@ public func discoveredClangToolInfo( var blocklists = ClangBlocklists() blocklists.caching = getBlocklist(type: ClangBlocklists.CachingBlocklistInfo.self, toolchainFilename: "clang-caching.json", delegate: delegate) blocklists.builtinModuleVerify = getBlocklist(type: ClangBlocklists.BuiltinModuleVerifierInfo.self, toolchainFilename: "clang-builtin-module-verify.json", delegate: delegate) - + blocklists.clangExplicitModules = getBlocklist(type: ClangBlocklists.ClangExplicitModulesInfo.self, toolchainFilename: "clang-explicit-modules.json", delegate: delegate) return DiscoveredClangToolSpecInfo( toolPath: toolPath, diff --git a/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift b/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift index 99f43dd8e..9e446bfe9 100644 --- a/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift +++ b/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift @@ -100,6 +100,7 @@ fileprivate struct ClangCompilationCachingTests: CoreBasedTests { "PRODUCT_NAME": "$(TARGET_NAME)", "CLANG_ENABLE_COMPILE_CACHE": "YES", "COMPILATION_CACHE_CAS_PATH": tmpDirPath.join("CompilationCache").str, + "COMPILATION_CACHE_ENABLE_DIAGNOSTIC_REMARKS": "YES", ])], targets: [ TestAggregateTarget( @@ -157,7 +158,9 @@ fileprivate struct ClangCompilationCachingTests: CoreBasedTests { try await tester.checkBuild(runDestination: .macOS, persistent: true) { results in _ = results.checkTask(.matchTargetName("Textual"), .matchRuleType("ScanDependencies")) { $0 } _ = results.checkTask(.matchTargetName("TextualWithFMod"), .matchRuleType("ScanDependencies")) { $0 } - results.checkNoTask(.matchTargetName("CppModules"), .matchRuleType("ScanDependencies")) + _ = results.checkTask(.matchTargetName("CppModules"), .matchRuleType("ScanDependencies")) { $0 } + let cppCompile: Task = try results.checkTask(.matchTargetName("CppModules"), .matchRuleType("CompileC")) { $0 } + results.checkCompileCacheMiss(cppCompile) } } } diff --git a/Tests/SWBTaskConstructionTests/ClangModulesTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/ClangModulesTaskConstructionTests.swift index 974c8d6ff..1e70429f9 100644 --- a/Tests/SWBTaskConstructionTests/ClangModulesTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/ClangModulesTaskConstructionTests.swift @@ -13,6 +13,7 @@ import Testing import SWBCore +import SWBMacro import SWBTestSupport import SWBUtil import SWBTaskConstruction @@ -78,4 +79,303 @@ fileprivate struct ClangModulesTaskConstructionTests: CoreBasedTests { } } } + + @Test(.requireSDKs(.macOS), arguments: [false, true]) + func cxxExplicitModulesOptIn(allowCxx: Bool) async throws { + try await withTemporaryDirectory { tmpDir in + let testProject = try await TestProject( + "aProject", + sourceRoot: tmpDir, + groupTree: TestGroup( + "SomeFiles", + children: [ + TestFile("a.c"), + TestFile("b.m"), + TestFile("c.cpp"), + TestFile("d.mm"), + ]), + buildConfigurations: [ + TestBuildConfiguration( + "Debug", + buildSettings: [ + "LIBTOOL": self.libtoolPath.str, + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_USE_RESPONSE_FILE": "NO", + "CLANG_ENABLE_MODULES": "YES", + "CLANG_ENABLE_EXPLICIT_MODULES": "YES", + "CC": clangCompilerPath.str, + "OTHER_CPLUSPLUSFLAGS": allowCxx ? "-fcxx-modules" : "" + ]), + ], + targets: [ + TestStandardTarget( + "Test", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase(["a.c", "b.m", "c.cpp", "d.mm"]), + ] + ) + ]) + + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + await tester.checkBuild(runDestination: .macOS) { results in + // C and ObjC always scan once explicit modules is enabled. + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("a.o")) + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("b.o")) + + if allowCxx { + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("c.o")) + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("d.o")) + } else { + results.checkNoTask(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("c.o")) + results.checkNoTask(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("d.o")) + } + + results.checkNoDiagnostics() + } + } + } + + /// Testing the EBM block list when caching is off and on. + @Test(.requireSDKs(.macOS), arguments: [false, true]) + func clangExplicitModulesBlocklist(compileCache: Bool) async throws { + let libtoolPath = try await self.libtoolPath + let clangPath = try await self.clangCompilerPath + try await withTemporaryDirectory { tmpDir in + let blocklistsDir = tmpDir.join("blocklists") + try localFS.createDirectory(blocklistsDir, recursive: true) + try await localFS.writeFileContents(blocklistsDir.join("clang-explicit-modules.json")) { stream in + stream <<< + """ + { "KnownFailures": ["aProject"] } + """ + } + + var buildSettings:[String: String] = [ + "LIBTOOL": libtoolPath.str, + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_USE_RESPONSE_FILE": "NO", + "CLANG_ENABLE_MODULES": "YES", + "CLANG_ENABLE_EXPLICIT_MODULES": "YES", + "OTHER_CPLUSPLUSFLAGS": "-fcxx-modules", + "BLOCKLISTS_PATH": blocklistsDir.str, + "CC": clangPath.str, + ] + + if compileCache { + buildSettings["CLANG_ENABLE_COMPILE_CACHE"] = "YES" + buildSettings["COMPILATION_CACHE_CAS_PATH"] = tmpDir.join("CompilationCache").str + } + + let testProject = TestProject( + "aProject", + sourceRoot: tmpDir, + groupTree: TestGroup( + "SomeFiles", + children: [ + TestFile("a.c"), + TestFile("b.m"), + TestFile("c.cpp"), + TestFile("d.mm"), + ]), + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: buildSettings)], + targets: [ + TestStandardTarget( + "Test", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase(["a.c", "b.m", "c.cpp", "d.mm"]), + ] + ) + ]) + + // Construct a custom core to test project identity based matching. + let core = try await Self.makeCore(registerExtraPlugins: { pluginManager in + struct TestSettingsBuilderExtension: SettingsBuilderExtension { + func matchesAnyProjectIdentities(scope: MacroEvaluationScope, projectIdentities: Set) -> Bool { + projectIdentities.contains(scope.evaluate(BuiltinMacros.PROJECT_NAME)) + } + } + pluginManager.register(TestSettingsBuilderExtension(), type: SettingsBuilderExtensionPoint.self) + }) + + let tester = try TaskConstructionTester(core, testProject) + await tester.checkBuild(runDestination: .macOS) { results in + results.checkNoTask(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("a.o")) + results.checkNoTask(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("b.o")) + results.checkNoTask(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("c.o")) + results.checkNoTask(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("d.o")) + + // Check that modules are still on. + results.checkTask(.matchRuleType("CompileC"), .matchRuleItemPattern(.suffix("a.c"))) { task in + task.checkCommandLineContains(["-fmodules"]) + // Caching requires explicit modules, so it has to be off. + task.checkCommandLineDoesNotContain("-fcas-backend") + } + results.checkTask(.matchRuleType("CompileC"), .matchRuleItemPattern(.suffix("c.cpp"))) { task in + task.checkCommandLineContains(["-fmodules"]) + task.checkCommandLineContains(["-fcxx-modules"]) + // Caching requires explicit modules, so it has to be off. + task.checkCommandLineDoesNotContain("-fcas-backend") + } + results.checkNoDiagnostics() + } + } + } + + /// Testing differet configurations that controls C++ explicit modules. + struct CxxModulesFlagCase: Sendable, CustomTestStringConvertible { + let name: String + let settings: [String: String] + let perFileArgs: [String]? + let expectScan: Bool + var testDescription: String { name } + } + + @Test(.requireSDKs(.macOS), arguments: [ + CxxModulesFlagCase(name: "OTHER_CFLAGS via OTHER_CPLUSPLUSFLAGS default", + settings: ["OTHER_CFLAGS": "-fcxx-modules"], perFileArgs: nil, expectScan: true), + CxxModulesFlagCase(name: "last flag wins, off", + settings: ["OTHER_CPLUSPLUSFLAGS": "-fcxx-modules -fno-cxx-modules"], perFileArgs: nil, expectScan: false), + CxxModulesFlagCase(name: "last flag wins, on", + settings: ["OTHER_CPLUSPLUSFLAGS": "-fno-cxx-modules -fcxx-modules"], perFileArgs: nil, expectScan: true), + CxxModulesFlagCase(name: "per-file overrides target", + settings: ["OTHER_CPLUSPLUSFLAGS": "-fno-cxx-modules"], perFileArgs: ["-fcxx-modules"], expectScan: true), + CxxModulesFlagCase(name: "CLANG_DISABLE_CXX_MODULES=NO", + settings: ["CLANG_DISABLE_CXX_MODULES": "NO"], perFileArgs: nil, expectScan: true), + ]) + func cxxHeaderModulesFlagResolution(_ testCase: CxxModulesFlagCase) async throws { + let libtoolPath = try await self.libtoolPath + let clangPath = try await self.clangCompilerPath + try await withTemporaryDirectory { tmpDir in + var buildSettings: [String: String] = [ + "LIBTOOL": libtoolPath.str, + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_USE_RESPONSE_FILE": "NO", + "CLANG_ENABLE_MODULES": "YES", + "CLANG_ENABLE_EXPLICIT_MODULES": "YES", + "CC": clangPath.str, + ] + buildSettings.merge(testCase.settings) { _, new in new } + + let testProject = TestProject( + "aProject", + sourceRoot: tmpDir, + groupTree: TestGroup("SomeFiles", children: [TestFile("c.cpp")]), + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: buildSettings)], + targets: [ + TestStandardTarget( + "Test", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase([TestBuildFile("c.cpp", additionalArgs: testCase.perFileArgs)]), + ]) + ]) + + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + await tester.checkBuild(runDestination: .macOS) { results in + if testCase.expectScan { + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("c.o")) + } else { + results.checkNoTask(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("c.o")) + } + results.checkNoDiagnostics() + } + } + } + + @Test(.requireSDKs(.macOS)) + func cxxHeaderModulesPerFileGranularity() async throws { + try await withTemporaryDirectory { tmpDir in + let testProject = try await TestProject( + "aProject", + sourceRoot: tmpDir, + groupTree: TestGroup("SomeFiles", children: [ + TestFile("a.c"), TestFile("c.cpp"), TestFile("d.mm"), + ]), + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: [ + "LIBTOOL": self.libtoolPath.str, + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_USE_RESPONSE_FILE": "NO", + "CLANG_ENABLE_MODULES": "YES", + "CLANG_ENABLE_EXPLICIT_MODULES": "YES", + "CC": clangCompilerPath.str, + ])], + targets: [ + TestStandardTarget( + "Test", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase([ + "a.c", + TestBuildFile("c.cpp", additionalArgs: ["-fcxx-modules"]), + "d.mm", + ]), + ]) + ]) + + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("a.o")) + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("c.o")) + results.checkNoTask(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("d.o")) + results.checkNoDiagnostics() + } + } + } + + + /// With caching enabled, C++/ObjC++ reach explicit modules via the caching path even when + /// `CLANG_ENABLE_EXPLICIT_MODULES` is NO. The "Compile caching is not supported + /// with implicit modules" warning should not fire for these sources. + @Test(.requireSDKs(.macOS)) + func cxxCachingUnaffectedWhenExplicitModulesDisabled() async throws { + try await withTemporaryDirectory { tmpDir in + let testProject = try await TestProject( + "aProject", + sourceRoot: tmpDir, + groupTree: TestGroup( + "SomeFiles", + children: [ + TestFile("c.cpp"), + TestFile("d.mm"), + ]), + buildConfigurations: [ + TestBuildConfiguration( + "Debug", + buildSettings: [ + "LIBTOOL": self.libtoolPath.str, + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_USE_RESPONSE_FILE": "NO", + "CLANG_ENABLE_MODULES": "YES", + "CLANG_ENABLE_EXPLICIT_MODULES": "NO", + "CLANG_ENABLE_COMPILE_CACHE": "YES", + "CC": clangCompilerPath.str, + ]), + ], + targets: [ + TestStandardTarget( + "Test", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase(["c.cpp", "d.mm"]), + ] + ) + ]) + + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + await tester.checkBuild(runDestination: .macOS) { results in + // C++/ObjC++ still scan when caching is on even when CLANG_ENABLE_EXPLICIT_MODULES is NO. + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("c.o")) + results.checkTaskExists(.matchRuleType("ScanDependencies"), .matchRuleItemBasename("d.o")) + // And no "Compile caching is not supported with implicit modules" warning. + results.checkNoDiagnostics() + } + } + } } +