Build: migrate to file lists for driver invocations

IntegrationTests/Tests/IntegrationTests/BasicTests.swift

@@ -215,7 +215,7 @@ final class BasicTests: XCTestCase {
 
             // Check the build.
             let buildOutput = try sh(swiftBuild, "--package-path", packagePath, "-v").stdout
-            XCTAssertMatch(buildOutput, .regex(#"swiftc.* -module-name special_tool .* '.*/more spaces/special tool/some file.swift'"#))
+            XCTAssertMatch(buildOutput, .regex(#"swiftc.* -module-name special_tool .* '@.*/more spaces/special tool/.build/[^/]+/debug/special_tool.build/sources'"#))
             XCTAssertMatch(buildOutput, .contains("Build complete"))
 
             // Verify that the tool exists and works.

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -74,6 +74,10 @@ public final class SwiftTargetBuildDescription {
         self.target.sources.paths + self.derivedSources.paths + self.pluginDerivedSources.paths
     }
 
+    public var sourcesFileListPath: AbsolutePath {
+        self.tempsPath.appending(component: "sources")
+    }
+
     /// The list of all resource files in the target, including the derived ones.
     public var resources: [Resource] {
         self.target.underlyingTarget.resources + self.pluginDerivedResources

Sources/Build/LLBuildManifestBuilder.swift

@@ -614,9 +614,10 @@ extension LLBuildManifestBuilder {
         let isLibrary = target.target.type == .library || target.target.type == .test
         let cmdName = target.target.getCommandName(config: self.buildConfig)
 
+        self.manifest.addWriteSourcesFileListCommand(sources: target.sources, sourcesFileListPath: target.sourcesFileListPath)
         self.manifest.addSwiftCmd(
             name: cmdName,
-            inputs: inputs,
+            inputs: inputs + [Node.file(target.sourcesFileListPath)],
             outputs: cmdOutputs,
             executable: self.buildParameters.toolchain.swiftCompilerPath,
             moduleName: target.target.c99name,
@@ -627,6 +628,7 @@ extension LLBuildManifestBuilder {
             objects: try target.objects,
             otherArguments: try target.compileArguments(),
             sources: target.sources,
+            fileList: target.sourcesFileListPath,
             isLibrary: isLibrary,
             wholeModuleOptimization: self.buildParameters.configuration == .release,
             outputFileMapPath: try target.writeOutputFileMap() // FIXME: Eliminate side effect.

Sources/LLBuildManifest/BuildManifest.swift

@@ -19,7 +19,7 @@ public protocol AuxiliaryFileType {
 }
 
 public enum WriteAuxiliary {
-    public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self]
+    public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self, SourcesFileList.self]
 
     public struct LinkFileList: AuxiliaryFileType {
         public static let name = "link-file-list"
@@ -50,6 +50,32 @@ public enum WriteAuxiliary {
             return content
         }
     }
+
+    public struct SourcesFileList: AuxiliaryFileType {
+        public static let name = "sources-file-list"
+
+        public static func computeInputs(sources: [AbsolutePath]) -> [Node] {
+            return [.virtual(Self.name)] + sources.map { Node.file($0) }
+        }
+
+        public static func getFileContents(inputs: [Node]) throws -> String {
+            let sources = inputs.compactMap {
+                if $0.kind == .file {
+                    return $0.name
+                } else {
+                    return nil
+                }
+            }
+
+            guard sources.count > 0 else { return "" }
+
+            var contents = sources
+                .map { $0.spm_shellEscaped() }
+                .joined(separator: "\n")
+            contents.append("\n")
+            return contents
+        }
+    }
 }
 
 public struct BuildManifest {
@@ -137,6 +163,16 @@ public struct BuildManifest {
         commands[name] = Command(name: name, tool: tool)
     }
 
+    public mutating func addWriteSourcesFileListCommand(
+        sources: [AbsolutePath],
+        sourcesFileListPath: AbsolutePath
+    ) {
+        let inputs = WriteAuxiliary.SourcesFileList.computeInputs(sources: sources)
+        let tool = WriteAuxiliaryFile(inputs: inputs, outputFilePath: sourcesFileListPath)
+        let name = sourcesFileListPath.pathString
+        commands[name] = Command(name: name, tool: tool)
+    }
+
     public mutating func addPkgStructureCmd(
         name: String,
         inputs: [Node],
@@ -222,6 +258,7 @@ public struct BuildManifest {
         objects: [AbsolutePath],
         otherArguments: [String],
         sources: [AbsolutePath],
+        fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
         outputFileMapPath: AbsolutePath
@@ -239,6 +276,7 @@ public struct BuildManifest {
             objects: objects,
             otherArguments: otherArguments,
             sources: sources,
+            fileList: fileList,
             isLibrary: isLibrary,
             wholeModuleOptimization: wholeModuleOptimization,
             outputFileMapPath: outputFileMapPath

Sources/LLBuildManifest/Tools.swift

@@ -262,6 +262,7 @@ public struct SwiftCompilerTool: ToolProtocol {
     public var objects: [AbsolutePath]
     public var otherArguments: [String]
     public var sources: [AbsolutePath]
+    public var fileList: AbsolutePath
     public var isLibrary: Bool
     public var wholeModuleOptimization: Bool
     public var outputFileMapPath: AbsolutePath
@@ -278,6 +279,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         objects: [AbsolutePath],
         otherArguments: [String],
         sources: [AbsolutePath],
+        fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
         outputFileMapPath: AbsolutePath
@@ -293,6 +295,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         self.objects = objects
         self.otherArguments = otherArguments
         self.sources = sources
+        self.fileList = fileList
         self.isLibrary = isLibrary
         self.wholeModuleOptimization = wholeModuleOptimization
         self.outputFileMapPath = outputFileMapPath
@@ -325,7 +328,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         if wholeModuleOptimization {
             arguments += ["-whole-module-optimization", "-num-threads", "\(Self.numThreads)"]
         }
-        arguments += ["-c"] + sources.map { $0.pathString }
+        arguments += ["-c", "@\(self.fileList.pathString)"]
         arguments += ["-I", importPath.pathString]
         arguments += otherArguments
         return arguments

Tests/BuildTests/BuildPlanTests.swift

@@ -877,7 +877,7 @@ final class BuildPlanTests: XCTestCase {
             try llbuild.generateManifest(at: yaml)
             let contents: String = try fs.readFileContents(yaml)
             XCTAssertMatch(contents, .contains("""
-                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "PkgLib.swiftmodule").escapedPathString())"]
+                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "PkgLib.swiftmodule").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"]
                 """))
 
         }
@@ -904,8 +904,9 @@ final class BuildPlanTests: XCTestCase {
             let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope)
             try llbuild.generateManifest(at: yaml)
             let contents: String = try fs.readFileContents(yaml)
+            let buildPath = plan.buildParameters.dataPath.appending(components: "debug")
             XCTAssertMatch(contents, .contains("""
-                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())"]
+                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"]
                 """))
         }
     }
@@ -3722,17 +3723,16 @@ final class BuildPlanTests: XCTestCase {
         let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope)
         try llbuild.generateManifest(at: yaml)
         let contents: String = try fs.readFileContents(yaml)
+
 #if os(Windows)
+        let suffix = ".exe"
+#else // FIXME(5472) - the suffix is dropped
+        let suffix = ""
+#endif
         XCTAssertMatch(contents, .contains("""
-                inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(buildPath.appending(components: "exe.exe").escapedPathString())"]
-                outputs: ["\(buildPath.appending(components: "swiftlib.build", "lib.swift.o").escapedPathString())","\(buildPath.escapedPathString())
-            """))
-#else   // FIXME(5472) - the suffix is dropped
-        XCTAssertMatch(contents, .contains("""
-                inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(buildPath.appending(components: "exe").escapedPathString())"]
+                inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(buildPath.appending(components: "exe\(suffix)").escapedPathString())","\(buildPath.appending(components: "swiftlib.build", "sources").escapedPathString())"]
                 outputs: ["\(buildPath.appending(components: "swiftlib.build", "lib.swift.o").escapedPathString())","\(buildPath.escapedPathString())
             """))
-#endif
     }
 
     func testObjCHeader1() throws {