From d09fc230b68ee0541436bedbf8eed2562b8a9cc5 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 31 Mar 2025 12:55:40 -0700 Subject: [PATCH] [Caching] Remap block-list files if needed Fix few remaining paths that are not re-mapped on the command-line when caching/prefix-mapping is enabled. rdar://147955190 --- Sources/SwiftDriver/Driver/Driver.swift | 20 +++++++++---- .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 29 +++++++++++-------- .../SwiftDriverTests/CachingBuildTests.swift | 10 ++++++- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index d6a93a670..1403a8cb5 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -314,7 +314,7 @@ public struct Driver { /// Code & data for incremental compilation. Nil if not running in incremental mode. /// Set during planning because needs the jobs to look at outputs. @_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil - + /// The graph of explicit module dependencies of this module, if the driver has planned an explicit module build. public private(set) var intermoduleDependencyGraph: InterModuleDependencyGraph? = nil @@ -703,6 +703,14 @@ public struct Driver { return try toolchain.lookupSwiftScanLib() } + func findBlocklists() throws -> [AbsolutePath] { + if let mockBlocklistDir = env["_SWIFT_DRIVER_MOCK_BLOCK_LIST_DIR"] { + // Use testing block-list directory. + return try Driver.findBlocklists(RelativeTo: try AbsolutePath(validating: mockBlocklistDir)) + } + return try Driver.findBlocklists(RelativeTo: try toolchain.executableDir) + } + @_spi(Testing) public static func findBlocklists(RelativeTo execDir: AbsolutePath) throws -> [AbsolutePath] { // Expect to find all blocklists in such dir: @@ -890,7 +898,7 @@ public struct Driver { let cwd = fileSystem.currentWorkingDirectory return try cwd.map{ try AbsolutePath(validating: workingDirectoryArg.asSingle, relativeTo: $0) } ?? AbsolutePath(validating: workingDirectoryArg.asSingle) } - + if let specifiedWorkingDir = self.workingDirectory { // Apply the working directory to the parsed options if passed explicitly. try Self.applyWorkingDirectory(specifiedWorkingDir, to: &self.parsedOptions) @@ -987,8 +995,8 @@ public struct Driver { self.lto = Self.ltoKind(&parsedOptions, diagnosticsEngine: diagnosticsEngine) // Figure out the primary outputs from the driver. - (self.compilerOutputType, self.linkerOutputType) = - Self.determinePrimaryOutputs(&parsedOptions, targetTriple: self.frontendTargetInfo.target.triple, + (self.compilerOutputType, self.linkerOutputType) = + Self.determinePrimaryOutputs(&parsedOptions, targetTriple: self.frontendTargetInfo.target.triple, driverKind: driverKind, diagnosticsEngine: diagnosticEngine) // Multithreading. @@ -1980,12 +1988,12 @@ extension Driver { guard let buildRecordInfo = self.buildRecordInfo, let incrementalCompilationState = self.incrementalCompilationState else { return } - + let buildRecord = buildRecordInfo.buildRecord( jobs, self.incrementalCompilationState?.blockingConcurrentMutationToProtectedState{ $0.skippedCompilationInputs }) - + do { try incrementalCompilationState.writeDependencyGraph(to: buildRecordInfo.dependencyGraphPath, buildRecord) } catch { diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index cbbe5e49f..b88c07ff3 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -188,9 +188,10 @@ extension Driver { } } - try commandLine.appendAll(.I, from: &parsedOptions) - try commandLine.appendAll(.F, .Fsystem, from: &parsedOptions) - try commandLine.appendAll(.vfsoverlay, from: &parsedOptions) + // TODO: Can we drop all search paths for compile jobs for explicit module build? + try addAllArgumentsWithPath(.I, to: &commandLine, remap: jobNeedPathRemap) + try addAllArgumentsWithPath(.F, .Fsystem, to: &commandLine, remap: jobNeedPathRemap) + try addAllArgumentsWithPath(.vfsoverlay, to: &commandLine, remap: jobNeedPathRemap) if let gccToolchain = parsedOptions.getLastArgument(.gccToolchain) { appendXccFlag("--gcc-toolchain=\(gccToolchain.asSingle)") @@ -247,8 +248,8 @@ extension Driver { commandLine.appendFlag("-sample-profile-use-profi") } try commandLine.appendAllExcept( - includeList: [.warningTreating], - excludeList: [], + includeList: [.warningTreating], + excludeList: [], from: &parsedOptions ) try commandLine.appendLast(.sanitizeEQ, from: &parsedOptions) @@ -350,9 +351,9 @@ extension Driver { } if isFrontendArgSupported(.blockListFile) { - try Driver.findBlocklists(RelativeTo: try toolchain.executableDir).forEach { + try findBlocklists().forEach { commandLine.appendFlag(.blockListFile) - commandLine.appendPath($0) + try addPathArgument(VirtualPath.absolute($0), to: &commandLine) } } @@ -1012,15 +1013,20 @@ extension Driver { } public mutating func addPathOption(option: Option, path: VirtualPath, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws { - commandLine.appendFlag(option) - let needRemap = remap && option.attributes.contains(.argumentIsPath) && + let needRemap = remap && isCachingEnabled && option.attributes.contains(.argumentIsPath) && !option.attributes.contains(.cacheInvariant) - try addPathArgument(path, to: &commandLine, remap: needRemap) + let commandPath = needRemap ? remapPath(path) : path + if option.kind == .joined { + commandLine.append(.joinedOptionAndPath(option.spelling, commandPath)) + } else { + // All other kinds that involves a path can be added as separated args. + commandLine.appendFlag(option) + commandLine.appendPath(commandPath) + } } /// Helper function to add last argument with path to command-line. public mutating func addLastArgumentWithPath(_ options: Option..., - from parsedOptions: inout ParsedOptions, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws { guard let parsedOption = parsedOptions.last(for: options) else { @@ -1031,7 +1037,6 @@ extension Driver { /// Helper function to add all arguments with path to command-line. public mutating func addAllArgumentsWithPath(_ options: Option..., - from parsedOptions: inout ParsedOptions, to commandLine: inout [Job.ArgTemplate], remap: Bool) throws { for matching in parsedOptions.arguments(for: options) { diff --git a/Tests/SwiftDriverTests/CachingBuildTests.swift b/Tests/SwiftDriverTests/CachingBuildTests.swift index e9052428d..eea09f1a8 100644 --- a/Tests/SwiftDriverTests/CachingBuildTests.swift +++ b/Tests/SwiftDriverTests/CachingBuildTests.swift @@ -895,6 +895,9 @@ final class CachingBuildTests: XCTestCase { .appending(component: "Swift") let casPath = path.appending(component: "cas") let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + let mockBlocklistDir = try testInputsPath.appending(components: "Dummy.xctoolchain", "usr", "bin") + var env = ProcessEnv.vars + env["_SWIFT_DRIVER_MOCK_BLOCK_LIST_DIR"] = mockBlocklistDir.nativePathString(escaped: true) var driver = try Driver(args: ["swiftc", "-I", cHeadersPath.nativePathString(escaped: true), "-I", swiftModuleInterfacesPath.nativePathString(escaped: true), @@ -907,6 +910,7 @@ final class CachingBuildTests: XCTestCase { "-scanner-prefix-map", testInputsPath.description + "=/^src", "-scanner-prefix-map", path.description + "=/^tmp", main.nativePathString(escaped: true)] + sdkArgumentsForTesting, + env: env, interModuleDependencyOracle: dependencyOracle) guard driver.isFrontendArgSupported(.scannerPrefixMap) else { throw XCTSkip("frontend doesn't support prefix map") @@ -929,7 +933,11 @@ final class CachingBuildTests: XCTestCase { // The only one that is not remapped should be the `-cas-path` that points to // `casPath`. XCTAssertFalse(command.contains { - return $0.starts(with: path.description) && $0 != casPath.description + $0.starts(with: path.description) && $0 != casPath.description + }) + /// All source location path should be remapped as well. + XCTAssertFalse(try command.contains { + $0.starts(with: try testInputsPath.description) }) } }