Write link file list as a build command (#6606)

This moves the generation of link file lists into the build system instead of doing it ad-hoc outside of the build. For this, we have a new `WriteAuxiliaryFile` tool and associated command that should be usable for any kind of writing of auxiliary files during the build.

Note that this change opted to not touch the existing infrastructure for in-process tools, so any inputs that are needed for the file generation will need to be flattened into a generic array of input nodes. The different types of file generation are keyed off a virtual node at the start of that array.

Author: neonichu

Sources/Build/BuildDescription/ProductBuildDescription.swift

@@ -336,20 +336,6 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
         return self.stripInvalidArguments(args)
     }
 
-    /// Writes link filelist to the filesystem.
-    func writeLinkFilelist(_ fs: FileSystem) throws {
-        var content = self.objects
-            .map { $0.pathString.spm_shellEscaped() }
-            .joined(separator: "\n")
-
-        // not sure this is needed, added here for backward compatibility
-        if !content.isEmpty {
-            content.append("\n")
-        }
-
-        try fs.writeFileContents(self.linkFileListPath, string: content)
-    }
-
     /// Returns the build flags from the declared build settings.
     private func buildSettingsFlags() -> [String] {
         var flags: [String] = []

Sources/Build/BuildOperation.swift

@@ -699,6 +699,7 @@ extension BuildDescription {
         let testDiscoveryCommands = llbuild.manifest.getCmdToolMap(kind: TestDiscoveryTool.self)
         let testEntryPointCommands = llbuild.manifest.getCmdToolMap(kind: TestEntryPointTool.self)
         let copyCommands = llbuild.manifest.getCmdToolMap(kind: CopyTool.self)
+        let writeCommands = llbuild.manifest.getCmdToolMap(kind: WriteAuxiliaryFile.self)
 
         // Create the build description.
         let buildDescription = try BuildDescription(
@@ -708,6 +709,7 @@ extension BuildDescription {
             testDiscoveryCommands: testDiscoveryCommands,
             testEntryPointCommands: testEntryPointCommands,
             copyCommands: copyCommands,
+            writeCommands: writeCommands,
             pluginDescriptions: plan.pluginDescriptions
         )
         try fileSystem.createDirectory(

Sources/Build/BuildOperationBuildSystemDelegateHandler.swift

@@ -303,6 +303,9 @@ public struct BuildDescription: Codable {
     /// The map of copy commands.
     let copyCommands: [BuildManifest.CmdName: LLBuildManifest.CopyTool]
 
+    /// The map of write commands.
+    let writeCommands: [BuildManifest.CmdName: LLBuildManifest.WriteAuxiliaryFile]
+
     /// A flag that indicates this build should perform a check for whether targets only import
     /// their explicitly-declared dependencies
     let explicitTargetDependencyImportCheckingMode: BuildParameters.TargetDependencyImportCheckingMode
@@ -329,13 +332,15 @@ public struct BuildDescription: Codable {
         testDiscoveryCommands: [BuildManifest.CmdName: LLBuildManifest.TestDiscoveryTool],
         testEntryPointCommands: [BuildManifest.CmdName: LLBuildManifest.TestEntryPointTool],
         copyCommands: [BuildManifest.CmdName: LLBuildManifest.CopyTool],
+        writeCommands: [BuildManifest.CmdName: LLBuildManifest.WriteAuxiliaryFile],
         pluginDescriptions: [PluginDescription]
     ) throws {
         self.swiftCommands = swiftCommands
         self.swiftFrontendCommands = swiftFrontendCommands
         self.testDiscoveryCommands = testDiscoveryCommands
         self.testEntryPointCommands = testEntryPointCommands
         self.copyCommands = copyCommands
+        self.writeCommands = writeCommands
         self.explicitTargetDependencyImportCheckingMode = plan.buildParameters
             .explicitTargetDependencyImportCheckingMode
         self.targetDependencyMap = try plan.targets.reduce(into: [TargetName: [TargetName]]()) {
@@ -465,6 +470,76 @@ public final class BuildExecutionContext {
     }
 }
 
+final class WriteAuxiliaryFileCommand: CustomLLBuildCommand {
+    override func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] {
+        guard let buildDescription = self.context.buildDescription else {
+            return []
+        }
+        guard let tool = buildDescription.copyCommands[command.name] else {
+            return []
+        }
+
+        do {
+            let encoder = JSONEncoder.makeWithDefaults()
+            var hash = Data()
+            hash += try encoder.encode(tool.inputs)
+            hash += try encoder.encode(tool.outputs)
+            return [UInt8](hash)
+        } catch {
+            self.context.observabilityScope.emit(error: "getSignature() failed: \(error.interpolationDescription)")
+            return []
+        }
+    }
+
+    override func execute(
+        _ command: SPMLLBuild.Command,
+        _: SPMLLBuild.BuildSystemCommandInterface
+    ) -> Bool {
+        let outputFilePath: AbsolutePath
+        let tool: WriteAuxiliaryFile!
+
+        do {
+            guard let buildDescription = self.context.buildDescription else {
+                throw InternalError("unknown build description")
+            }
+            guard let _tool = buildDescription.writeCommands[command.name] else {
+                throw StringError("command \(command.name) not registered")
+            }
+            tool = _tool
+
+            guard let output = tool.outputs.first, output.kind == .file else {
+                throw StringError("invalid output path")
+            }
+            outputFilePath = try AbsolutePath(validating: output.name)
+        } catch {
+            self.context.observabilityScope.emit(error: "failed to write auxiliary file: \(error.interpolationDescription)")
+            return false
+        }
+
+        do {
+            try self.context.fileSystem.writeFileContents(outputFilePath, string: getFileContents(tool: tool))
+            return true
+        } catch {
+            self.context.observabilityScope.emit(error: "failed to write auxiliary file '\(outputFilePath.pathString)': \(error.interpolationDescription)")
+            return false
+        }
+    }
+
+    func getFileContents(tool: WriteAuxiliaryFile) throws -> String {
+        guard tool.inputs.first?.kind == .virtual, let generatedFileType = tool.inputs.first?.name.dropFirst().dropLast() else {
+            throw StringError("invalid inputs")
+        }
+
+        for fileType in WriteAuxiliary.fileTypes {
+            if generatedFileType == fileType.name {
+                return try fileType.getFileContents(inputs: Array(tool.inputs.dropFirst()))
+            }
+        }
+
+        throw InternalError("unhandled generated file type '\(generatedFileType)'")
+    }
+}
+
 public protocol PackageStructureDelegate {
     func packageStructureChanged() -> Bool
 }
@@ -584,6 +659,8 @@ final class BuildOperationBuildSystemDelegateHandler: LLBuildBuildSystemDelegate
             return InProcessTool(buildExecutionContext, type: PackageStructureCommand.self)
         case CopyTool.name:
             return InProcessTool(buildExecutionContext, type: CopyCommand.self)
+        case WriteAuxiliaryFile.name:
+            return InProcessTool(buildExecutionContext, type: WriteAuxiliaryFileCommand.self)
         default:
             return nil
         }

Sources/Build/BuildPlan.swift

@@ -682,12 +682,6 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
         }
         buildProduct.libraryBinaryPaths = dependencies.libraryBinaryPaths
 
-        // Write the link filelist file.
-        //
-        // FIXME: We should write this as a custom llbuild task once we adopt it
-        // as a library.
-        try buildProduct.writeLinkFilelist(fileSystem)
-
         buildProduct.availableTools = dependencies.availableTools
     }
 

Sources/Build/LLBuildManifestBuilder.swift

@@ -989,13 +989,13 @@ extension LLBuildManifestBuilder {
             try self.manifest.addShellCmd(
                 name: cmdName,
                 description: "Archiving \(buildProduct.binaryPath.prettyPath())",
-                inputs: buildProduct.objects.map(Node.file),
+                inputs: (buildProduct.objects + [buildProduct.linkFileListPath]).map(Node.file),
                 outputs: [.file(buildProduct.binaryPath)],
                 arguments: try buildProduct.archiveArguments()
             )
 
         default:
-            let inputs = try buildProduct.objects + buildProduct.dylibs.map{ try $0.binaryPath }
+            let inputs = try buildProduct.objects + buildProduct.dylibs.map{ try $0.binaryPath } + [buildProduct.linkFileListPath]
 
             try self.manifest.addShellCmd(
                 name: cmdName,
@@ -1023,6 +1023,8 @@ extension LLBuildManifestBuilder {
             }
             self.addNode(output, toTarget: .test)
         }
+
+        self.manifest.addWriteLinkFileListCommand(objects: Array(buildProduct.objects), linkFileListPath: buildProduct.linkFileListPath)
     }
 }
 

Sources/LLBuildManifest/BuildManifest.swift

@@ -12,6 +12,46 @@
 
 import Basics
 
+public protocol AuxiliaryFileType {
+    static var name: String { get }
+
+    static func getFileContents(inputs: [Node]) throws -> String
+}
+
+public enum WriteAuxiliary {
+    public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self]
+
+    public struct LinkFileList: AuxiliaryFileType {
+        public static let name = "link-file-list"
+
+        // FIXME: We should extend the `InProcessTool` support to allow us to specify these in a typed way, but today we have to flatten all the inputs into a generic `Node` array (rdar://109844243).
+        public static func computeInputs(objects: [AbsolutePath]) -> [Node] {
+            return [.virtual(Self.name)] + objects.map { Node.file($0) }
+        }
+
+        public static func getFileContents(inputs: [Node]) throws -> String {
+            let objects = inputs.compactMap {
+                if $0.kind == .file {
+                    return $0.name
+                } else {
+                    return nil
+                }
+            }
+
+            var content = objects
+                .map { $0.spm_shellEscaped() }
+                .joined(separator: "\n")
+
+            // not sure this is needed, added here for backward compatibility
+            if !content.isEmpty {
+                content.append("\n")
+            }
+
+            return content
+        }
+    }
+}
+
 public struct BuildManifest {
     public typealias TargetName = String
     public typealias CmdName = String
@@ -87,6 +127,16 @@ public struct BuildManifest {
         commands[name] = Command(name: name, tool: tool)
     }
 
+    public mutating func addWriteLinkFileListCommand(
+        objects: [AbsolutePath],
+        linkFileListPath: AbsolutePath
+    ) {
+        let inputs = WriteAuxiliary.LinkFileList.computeInputs(objects: objects)
+        let tool = WriteAuxiliaryFile(inputs: inputs, outputFilePath: linkFileListPath)
+        let name = linkFileListPath.pathString
+        commands[name] = Command(name: name, tool: tool)
+    }
+
     public mutating func addPkgStructureCmd(
         name: String,
         inputs: [Node],

Sources/LLBuildManifest/Tools.swift

@@ -150,6 +150,26 @@ public struct ShellTool: ToolProtocol {
     }
 }
 
+public struct WriteAuxiliaryFile: ToolProtocol {
+    public static let name: String = "write-auxiliary-file"
+
+    public let inputs: [Node]
+    private let outputFilePath: AbsolutePath
+
+    public init(inputs: [Node], outputFilePath: AbsolutePath) {
+        self.inputs = inputs
+        self.outputFilePath = outputFilePath
+    }
+
+    public var outputs: [Node] {
+        return [.file(outputFilePath)]
+    }
+
+    public func write(to stream: ManifestToolStream) {
+        stream["description"] = "Write auxiliary file \(outputFilePath.pathString)"
+    }
+}
+
 public struct ClangTool: ToolProtocol {
     public static let name: String = "clang"