Build: migrate to file lists for driver invocations

When invoking the driver use file lists rather than passing all the
sources on the command line. This is particularly import for Windows
where the command line limit is 32k. With deep dependency sets and large
projects, it is possible to overrun this limit and fail to invoke the
command. The driver itself will invoke the frontend using response files
if the invocation will overflow the command line limit.

Fixes: #6402

Author: compnerd

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -74,6 +74,10 @@ public final class SwiftTargetBuildDescription {
         self.target.sources.paths + self.derivedSources.paths + self.pluginDerivedSources.paths
     }
 
+    public var sourcesFileListPath: AbsolutePath {
+        self.tempsPath.appending(component: "sources")
+    }
+
     /// The list of all resource files in the target, including the derived ones.
     public var resources: [Resource] {
         self.target.underlyingTarget.resources + self.pluginDerivedResources

Sources/Build/LLBuildManifestBuilder.swift

@@ -614,9 +614,10 @@ extension LLBuildManifestBuilder {
         let isLibrary = target.target.type == .library || target.target.type == .test
         let cmdName = target.target.getCommandName(config: self.buildConfig)
 
+        self.manifest.addWriteSourcesFileListCommand(sources: target.sources, sourcesFileListPath: target.sourcesFileListPath)
         self.manifest.addSwiftCmd(
             name: cmdName,
-            inputs: inputs,
+            inputs: inputs + [Node.file(target.sourcesFileListPath)],
             outputs: cmdOutputs,
             executable: self.buildParameters.toolchain.swiftCompilerPath,
             moduleName: target.target.c99name,
@@ -627,6 +628,7 @@ extension LLBuildManifestBuilder {
             objects: try target.objects,
             otherArguments: try target.compileArguments(),
             sources: target.sources,
+            fileList: target.sourcesFileListPath,
             isLibrary: isLibrary,
             wholeModuleOptimization: self.buildParameters.configuration == .release,
             outputFileMapPath: try target.writeOutputFileMap() // FIXME: Eliminate side effect.

Sources/LLBuildManifest/BuildManifest.swift

@@ -19,7 +19,7 @@ public protocol AuxiliaryFileType {
 }
 
 public enum WriteAuxiliary {
-    public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self]
+    public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self, SourcesFileList.self]
 
     public struct LinkFileList: AuxiliaryFileType {
         public static let name = "link-file-list"
@@ -50,6 +50,32 @@ public enum WriteAuxiliary {
             return content
         }
     }
+
+    public struct SourcesFileList: AuxiliaryFileType {
+        public static let name = "sources-file-list"
+
+        public static func computeInputs(sources: [AbsolutePath]) -> [Node] {
+            return [.virtual(Self.name)] + sources.map { Node.file($0) }
+        }
+
+        public static func getFileContents(inputs: [Node]) throws -> String {
+            let sources = inputs.compactMap {
+                if $0.kind == .file {
+                    return $0.name
+                } else {
+                    return nil
+                }
+            }
+
+            guard sources.count > 0 else { return "" }
+
+            var contents = sources
+                .map { $0.spm_shellEscaped() }
+                .joined(separator: "\n")
+            contents.append("\n")
+            return contents
+        }
+    }
 }
 
 public struct BuildManifest {
@@ -137,6 +163,16 @@ public struct BuildManifest {
         commands[name] = Command(name: name, tool: tool)
     }
 
+    public mutating func addWriteSourcesFileListCommand(
+        sources: [AbsolutePath],
+        sourcesFileListPath: AbsolutePath
+    ) {
+        let inputs = WriteAuxiliary.SourcesFileList.computeInputs(sources: sources)
+        let tool = WriteAuxiliaryFile(inputs: inputs, outputFilePath: sourcesFileListPath)
+        let name = sourcesFileListPath.pathString
+        commands[name] = Command(name: name, tool: tool)
+    }
+
     public mutating func addPkgStructureCmd(
         name: String,
         inputs: [Node],
@@ -222,6 +258,7 @@ public struct BuildManifest {
         objects: [AbsolutePath],
         otherArguments: [String],
         sources: [AbsolutePath],
+        fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
         outputFileMapPath: AbsolutePath
@@ -239,6 +276,7 @@ public struct BuildManifest {
             objects: objects,
             otherArguments: otherArguments,
             sources: sources,
+            fileList: fileList,
             isLibrary: isLibrary,
             wholeModuleOptimization: wholeModuleOptimization,
             outputFileMapPath: outputFileMapPath

Sources/LLBuildManifest/Tools.swift

@@ -262,6 +262,7 @@ public struct SwiftCompilerTool: ToolProtocol {
     public var objects: [AbsolutePath]
     public var otherArguments: [String]
     public var sources: [AbsolutePath]
+    public var fileList: AbsolutePath
     public var isLibrary: Bool
     public var wholeModuleOptimization: Bool
     public var outputFileMapPath: AbsolutePath
@@ -278,6 +279,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         objects: [AbsolutePath],
         otherArguments: [String],
         sources: [AbsolutePath],
+        fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
         outputFileMapPath: AbsolutePath
@@ -293,6 +295,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         self.objects = objects
         self.otherArguments = otherArguments
         self.sources = sources
+        self.fileList = fileList
         self.isLibrary = isLibrary
         self.wholeModuleOptimization = wholeModuleOptimization
         self.outputFileMapPath = outputFileMapPath
@@ -325,7 +328,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         if wholeModuleOptimization {
             arguments += ["-whole-module-optimization", "-num-threads", "\(Self.numThreads)"]
         }
-        arguments += ["-c"] + sources.map { $0.pathString }
+        arguments += ["-c", "@\(self.fileList.pathString)"]
         arguments += ["-I", importPath.pathString]
         arguments += otherArguments
         return arguments

Tests/BuildTests/BuildPlanTests.swift

@@ -877,7 +877,7 @@ final class BuildPlanTests: XCTestCase {
             try llbuild.generateManifest(at: yaml)
             let contents: String = try fs.readFileContents(yaml)
             XCTAssertMatch(contents, .contains("""
-                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "PkgLib.swiftmodule").escapedPathString())"]
+                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "PkgLib.swiftmodule").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"]
                 """))
 
         }
@@ -904,8 +904,9 @@ final class BuildPlanTests: XCTestCase {
             let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope)
             try llbuild.generateManifest(at: yaml)
             let contents: String = try fs.readFileContents(yaml)
+            let buildPath = plan.buildParameters.dataPath.appending(components: "debug")
             XCTAssertMatch(contents, .contains("""
-                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())"]
+                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"]
                 """))
         }
     }
@@ -3722,17 +3723,16 @@ final class BuildPlanTests: XCTestCase {
         let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope)
         try llbuild.generateManifest(at: yaml)
         let contents: String = try fs.readFileContents(yaml)
+
 #if os(Windows)
+        let suffix = ".exe"
+#else // FIXME(5472) - the suffix is dropped
+        let suffix = ""
+#endif
         XCTAssertMatch(contents, .contains("""
-                inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(buildPath.appending(components: "exe.exe").escapedPathString())"]
-                outputs: ["\(buildPath.appending(components: "swiftlib.build", "lib.swift.o").escapedPathString())","\(buildPath.escapedPathString())
-            """))
-#else   // FIXME(5472) - the suffix is dropped
-        XCTAssertMatch(contents, .contains("""
-                inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(buildPath.appending(components: "exe").escapedPathString())"]
+                inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(buildPath.appending(components: "exe\(suffix)").escapedPathString())","\(buildPath.appending(components: "swiftlib.build", "sources").escapedPathString())"]
                 outputs: ["\(buildPath.appending(components: "swiftlib.build", "lib.swift.o").escapedPathString())","\(buildPath.escapedPathString())
             """))
-#endif
     }
 
     func testObjCHeader1() throws {