Skip to content

Commit ca781f6

Browse files
committed
Write link file list as a build command
This moves the generation of link file lists into the build system instead of doing it ad-hoc outside of the build. For this, we have a new `WriteAuxiliaryFile` tool and associated command that should be usable for any kind of writing of auxiliary files during the build. Note that this change opted to not touch the existing infrastructure for in-process tools, so any inputs that are needed for the file generation will need to be flattened into a generic array of input nodes. The different types of file generation are keyed off a virtual node at the start of that array. (cherry picked from commit ee7f064)
1 parent 9be12c5 commit ca781f6

8 files changed

+230
-94
lines changed

Sources/Build/BuildDescription/ProductBuildDescription.swift

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -339,18 +339,6 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
339339
return self.stripInvalidArguments(args)
340340
}
341341

342-
/// Writes link filelist to the filesystem.
343-
func writeLinkFilelist(_ fs: FileSystem) throws {
344-
let stream = BufferedOutputByteStream()
345-
346-
for object in self.objects {
347-
stream <<< object.pathString.spm_shellEscaped() <<< "\n"
348-
}
349-
350-
try fs.createDirectory(self.linkFileListPath.parentDirectory, recursive: true)
351-
try fs.writeFileContents(self.linkFileListPath, bytes: stream.bytes)
352-
}
353-
354342
/// Returns the build flags from the declared build settings.
355343
private func buildSettingsFlags() -> [String] {
356344
var flags: [String] = []

Sources/Build/BuildOperation.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -694,6 +694,7 @@ extension BuildDescription {
694694
let testDiscoveryCommands = llbuild.manifest.getCmdToolMap(kind: TestDiscoveryTool.self)
695695
let testEntryPointCommands = llbuild.manifest.getCmdToolMap(kind: TestEntryPointTool.self)
696696
let copyCommands = llbuild.manifest.getCmdToolMap(kind: CopyTool.self)
697+
let writeCommands = llbuild.manifest.getCmdToolMap(kind: WriteAuxiliaryFile.self)
697698

698699
// Create the build description.
699700
let buildDescription = try BuildDescription(
@@ -703,6 +704,7 @@ extension BuildDescription {
703704
testDiscoveryCommands: testDiscoveryCommands,
704705
testEntryPointCommands: testEntryPointCommands,
705706
copyCommands: copyCommands,
707+
writeCommands: writeCommands,
706708
pluginDescriptions: plan.pluginDescriptions
707709
)
708710
try fileSystem.createDirectory(

Sources/Build/BuildOperationBuildSystemDelegateHandler.swift

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,9 @@ public struct BuildDescription: Codable {
297297
/// The map of copy commands.
298298
let copyCommands: [BuildManifest.CmdName: LLBuildManifest.CopyTool]
299299

300+
/// The map of write commands.
301+
let writeCommands: [BuildManifest.CmdName: LLBuildManifest.WriteAuxiliaryFile]
302+
300303
/// A flag that inidcates this build should perform a check for whether targets only import
301304
/// their explicitly-declared dependencies
302305
let explicitTargetDependencyImportCheckingMode: BuildParameters.TargetDependencyImportCheckingMode
@@ -323,13 +326,15 @@ public struct BuildDescription: Codable {
323326
testDiscoveryCommands: [BuildManifest.CmdName: LLBuildManifest.TestDiscoveryTool],
324327
testEntryPointCommands: [BuildManifest.CmdName: LLBuildManifest.TestEntryPointTool],
325328
copyCommands: [BuildManifest.CmdName: LLBuildManifest.CopyTool],
329+
writeCommands: [BuildManifest.CmdName: LLBuildManifest.WriteAuxiliaryFile],
326330
pluginDescriptions: [PluginDescription]
327331
) throws {
328332
self.swiftCommands = swiftCommands
329333
self.swiftFrontendCommands = swiftFrontendCommands
330334
self.testDiscoveryCommands = testDiscoveryCommands
331335
self.testEntryPointCommands = testEntryPointCommands
332336
self.copyCommands = copyCommands
337+
self.writeCommands = writeCommands
333338
self.explicitTargetDependencyImportCheckingMode = plan.buildParameters
334339
.explicitTargetDependencyImportCheckingMode
335340
self.targetDependencyMap = try plan.targets.reduce(into: [TargetName: [TargetName]]()) {
@@ -458,6 +463,76 @@ public final class BuildExecutionContext {
458463
}
459464
}
460465

466+
final class WriteAuxiliaryFileCommand: CustomLLBuildCommand {
467+
override func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] {
468+
guard let buildDescription = self.context.buildDescription else {
469+
return []
470+
}
471+
guard let tool = buildDescription.copyCommands[command.name] else {
472+
return []
473+
}
474+
475+
do {
476+
let encoder = JSONEncoder.makeWithDefaults()
477+
var hash = Data()
478+
hash += try encoder.encode(tool.inputs)
479+
hash += try encoder.encode(tool.outputs)
480+
return [UInt8](hash)
481+
} catch {
482+
self.context.observabilityScope.emit(error: "getSignature() failed: \(error.interpolationDescription)")
483+
return []
484+
}
485+
}
486+
487+
override func execute(
488+
_ command: SPMLLBuild.Command,
489+
_: SPMLLBuild.BuildSystemCommandInterface
490+
) -> Bool {
491+
let outputFilePath: AbsolutePath
492+
let tool: WriteAuxiliaryFile!
493+
494+
do {
495+
guard let buildDescription = self.context.buildDescription else {
496+
throw InternalError("unknown build description")
497+
}
498+
guard let _tool = buildDescription.writeCommands[command.name] else {
499+
throw StringError("command \(command.name) not registered")
500+
}
501+
tool = _tool
502+
503+
guard let output = tool.outputs.first, output.kind == .file else {
504+
throw StringError("invalid output path")
505+
}
506+
outputFilePath = try AbsolutePath(validating: output.name)
507+
} catch {
508+
self.context.observabilityScope.emit(error: "failed to write auxiliary file: \(error.interpolationDescription)")
509+
return false
510+
}
511+
512+
do {
513+
try self.context.fileSystem.writeFileContents(outputFilePath, string: getFileContents(tool: tool))
514+
return true
515+
} catch {
516+
self.context.observabilityScope.emit(error: "failed to write auxiliary file '\(outputFilePath.pathString)': \(error.interpolationDescription)")
517+
return false
518+
}
519+
}
520+
521+
func getFileContents(tool: WriteAuxiliaryFile) throws -> String {
522+
guard tool.inputs.first?.kind == .virtual, let generatedFileType = tool.inputs.first?.name.dropFirst().dropLast() else {
523+
throw StringError("invalid inputs")
524+
}
525+
526+
for fileType in WriteAuxiliary.fileTypes {
527+
if generatedFileType == fileType.name {
528+
return try fileType.getFileContents(inputs: Array(tool.inputs.dropFirst()))
529+
}
530+
}
531+
532+
throw InternalError("unhandled generated file type '\(generatedFileType)'")
533+
}
534+
}
535+
461536
public protocol PackageStructureDelegate {
462537
func packageStructureChanged() -> Bool
463538
}
@@ -577,6 +652,8 @@ final class BuildOperationBuildSystemDelegateHandler: LLBuildBuildSystemDelegate
577652
return InProcessTool(buildExecutionContext, type: PackageStructureCommand.self)
578653
case CopyTool.name:
579654
return InProcessTool(buildExecutionContext, type: CopyCommand.self)
655+
case WriteAuxiliaryFile.name:
656+
return InProcessTool(buildExecutionContext, type: WriteAuxiliaryFileCommand.self)
580657
default:
581658
return nil
582659
}

Sources/Build/BuildPlan.swift

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -694,12 +694,6 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
694694
}
695695
buildProduct.libraryBinaryPaths = dependencies.libraryBinaryPaths
696696

697-
// Write the link filelist file.
698-
//
699-
// FIXME: We should write this as a custom llbuild task once we adopt it
700-
// as a library.
701-
try buildProduct.writeLinkFilelist(fileSystem)
702-
703697
buildProduct.availableTools = dependencies.availableTools
704698
}
705699

Sources/Build/LLBuildManifestBuilder.swift

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -986,13 +986,13 @@ extension LLBuildManifestBuilder {
986986
try self.manifest.addShellCmd(
987987
name: cmdName,
988988
description: "Archiving \(buildProduct.binaryPath.prettyPath())",
989-
inputs: buildProduct.objects.map(Node.file),
989+
inputs: (buildProduct.objects + [buildProduct.linkFileListPath]).map(Node.file),
990990
outputs: [.file(buildProduct.binaryPath)],
991991
arguments: try buildProduct.archiveArguments()
992992
)
993993

994994
default:
995-
let inputs = try buildProduct.objects + buildProduct.dylibs.map{ try $0.binaryPath }
995+
let inputs = try buildProduct.objects + buildProduct.dylibs.map{ try $0.binaryPath } + [buildProduct.linkFileListPath]
996996

997997
try self.manifest.addShellCmd(
998998
name: cmdName,
@@ -1020,6 +1020,8 @@ extension LLBuildManifestBuilder {
10201020
}
10211021
self.addNode(output, toTarget: .test)
10221022
}
1023+
1024+
self.manifest.addWriteLinkFileListCommand(objects: Array(buildProduct.objects), linkFileListPath: buildProduct.linkFileListPath)
10231025
}
10241026
}
10251027

Sources/LLBuildManifest/BuildManifest.swift

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,46 @@
1313
import Basics
1414
import struct TSCBasic.AbsolutePath
1515

16+
public protocol AuxiliaryFileType {
17+
static var name: String { get }
18+
19+
static func getFileContents(inputs: [Node]) throws -> String
20+
}
21+
22+
public enum WriteAuxiliary {
23+
public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self]
24+
25+
public struct LinkFileList: AuxiliaryFileType {
26+
public static let name = "link-file-list"
27+
28+
// FIXME: We should extend the `InProcessTool` support to allow us to specify these in a typed way, but today we have to flatten all the inputs into a generic `Node` array (rdar://109844243).
29+
public static func computeInputs(objects: [AbsolutePath]) -> [Node] {
30+
return [.virtual(Self.name)] + objects.map { Node.file($0) }
31+
}
32+
33+
public static func getFileContents(inputs: [Node]) throws -> String {
34+
let objects = inputs.compactMap {
35+
if $0.kind == .file {
36+
return $0.name
37+
} else {
38+
return nil
39+
}
40+
}
41+
42+
var content = objects
43+
.map { $0.spm_shellEscaped() }
44+
.joined(separator: "\n")
45+
46+
// not sure this is needed, added here for backward compatibility
47+
if !content.isEmpty {
48+
content.append("\n")
49+
}
50+
51+
return content
52+
}
53+
}
54+
}
55+
1656
public struct BuildManifest {
1757
public typealias TargetName = String
1858
public typealias CmdName = String
@@ -88,6 +128,16 @@ public struct BuildManifest {
88128
commands[name] = Command(name: name, tool: tool)
89129
}
90130

131+
public mutating func addWriteLinkFileListCommand(
132+
objects: [AbsolutePath],
133+
linkFileListPath: AbsolutePath
134+
) {
135+
let inputs = WriteAuxiliary.LinkFileList.computeInputs(objects: objects)
136+
let tool = WriteAuxiliaryFile(inputs: inputs, outputFilePath: linkFileListPath)
137+
let name = linkFileListPath.pathString
138+
commands[name] = Command(name: name, tool: tool)
139+
}
140+
91141
public mutating func addPkgStructureCmd(
92142
name: String,
93143
inputs: [Node],

Sources/LLBuildManifest/Tools.swift

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,26 @@ public struct ShellTool: ToolProtocol {
151151
}
152152
}
153153

154+
public struct WriteAuxiliaryFile: ToolProtocol {
155+
public static let name: String = "write-auxiliary-file"
156+
157+
public let inputs: [Node]
158+
private let outputFilePath: AbsolutePath
159+
160+
public init(inputs: [Node], outputFilePath: AbsolutePath) {
161+
self.inputs = inputs
162+
self.outputFilePath = outputFilePath
163+
}
164+
165+
public var outputs: [Node] {
166+
return [.file(outputFilePath)]
167+
}
168+
169+
public func write(to stream: ManifestToolStream) {
170+
stream["description"] = "Write auxiliary file \(outputFilePath.pathString)"
171+
}
172+
}
173+
154174
public struct ClangTool: ToolProtocol {
155175
public static let name: String = "clang"
156176

0 commit comments

Comments
 (0)