Recipe: Get rid of all use of LinuxDistribution in SDKGenerator

Sources/GeneratorCLI/GeneratorCLI.swift

@@ -105,22 +105,31 @@ struct GeneratorCLI: AsyncParsableCommand {
 
     let elapsed = try await ContinuousClock().measure {
       let logger = Logger(label: "org.swift.swift-sdk-generator")
+      let hostTriple = try await SwiftSDKGenerator.getHostTriple(explicitArch: hostArch, isVerbose: verbose)
+      let targetTriple = Triple(
+        cpu: targetArch ?? hostTriple.cpu,
+        vendor: .unknown,
+        os: .linux,
+        environment: .gnu
+      )
+      let recipe = try LinuxRecipe(
+        targetTriple: targetTriple,
+        linuxDistribution: linuxDistribution,
+        swiftVersion: swiftVersion,
+        swiftBranch: swiftBranch,
+        lldVersion: lldVersion,
+        withDocker: withDocker,
+        fromContainerImage: fromContainerImage
+      )
       let generator = try await SwiftSDKGenerator(
         bundleVersion: self.bundleVersion,
-        hostCPUArchitecture: self.hostArch,
-        targetCPUArchitecture: self.targetArch,
-        swiftVersion: self.swiftVersion,
-        swiftBranch: self.swiftBranch,
-        lldVersion: self.lldVersion,
-        linuxDistribution: linuxDistribution,
-        shouldUseDocker: self.withDocker,
-        baseDockerImage: self.fromContainerImage,
-        artifactID: self.sdkName,
+        hostTriple: hostTriple,
+        targetTriple: targetTriple,
+        artifactID: self.sdkName ?? recipe.defaultArtifactID,
         isIncremental: self.incremental,
         isVerbose: self.verbose,
         logger: logger
       )
-      let recipe = LinuxRecipe()
 
       let serviceGroup = ServiceGroup(
         configuration: .init(

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift

@@ -13,13 +13,11 @@
 import SystemPackage
 
 extension SwiftSDKGenerator {
-  func copyTargetSwiftFromDocker() async throws {
+  func copyTargetSwiftFromDocker(targetDistribution: LinuxDistribution, baseDockerImage: String, sdkDirPath: FilePath) async throws {
     logGenerationStep("Launching a Docker container to copy Swift SDK for the target triple from it...")
-    try await withDockerContainer(fromImage: baseDockerImage!) { containerID in
-      let pathsConfiguration = self.pathsConfiguration
-
+    try await withDockerContainer(fromImage: baseDockerImage) { containerID in
       try await inTemporaryDirectory { generator, _ in
-        let sdkUsrPath = pathsConfiguration.sdkDirPath.appending("usr")
+        let sdkUsrPath = sdkDirPath.appending("usr")
         let sdkUsrLibPath = sdkUsrPath.appending("lib")
         try await generator.createDirectoryIfNeeded(at: sdkUsrPath)
         try await generator.copyFromDockerContainer(
@@ -28,7 +26,7 @@ extension SwiftSDKGenerator {
           to: sdkUsrPath.appending("include")
         )
 
-        if case .rhel = self.versionsConfiguration.linuxDistribution {
+        if case .rhel = targetDistribution {
           try await generator.runOnDockerContainer(
             id: containerID,
             command: #"""
@@ -56,7 +54,7 @@ extension SwiftSDKGenerator {
             from: containerLib64,
             to: sdkUsrLib64Path
           )
-          try await createSymlink(at: pathsConfiguration.sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64")
+          try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64")
         }
 
         try await generator.createDirectoryIfNeeded(at: sdkUsrLibPath)
@@ -68,7 +66,7 @@ extension SwiftSDKGenerator {
         // architecture-specific directories:
         //   https://wiki.ubuntu.com/MultiarchSpec
         // But not in all containers, so don't fail if it does not exist.
-        if case .ubuntu = self.versionsConfiguration.linuxDistribution {
+        if case .ubuntu = targetDistribution {
           subpaths += [("\(targetTriple.cpu)-linux-gnu", false)]
         }
 
@@ -80,27 +78,27 @@ extension SwiftSDKGenerator {
             failIfNotExists: failIfNotExists
           )
         }
-        try await generator.createSymlink(at: pathsConfiguration.sdkDirPath.appending("lib"), pointingTo: "usr/lib")
+        try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib")
 
         // Python artifacts are redundant.
         try await generator.removeRecursively(at: sdkUsrLibPath.appending("python3.10"))
 
         try await generator.removeRecursively(at: sdkUsrLibPath.appending("ssl"))
-        try await generator.copyTargetSwift(from: sdkUsrLibPath)
+        try await generator.copyTargetSwift(from: sdkUsrLibPath, sdkDirPath: sdkDirPath)
       }
     }
   }
 
-  func copyTargetSwift(from distributionPath: FilePath) async throws {
+  func copyTargetSwift(from distributionPath: FilePath, sdkDirPath: FilePath) async throws {
     logGenerationStep("Copying Swift core libraries for the target triple into Swift SDK bundle...")
 
     for (pathWithinPackage, pathWithinSwiftSDK) in [
       ("swift/linux", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift")),
       ("swift_static/linux", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static")),
       ("swift_static/shims", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static")),
-      ("swift/dispatch", pathsConfiguration.sdkDirPath.appending("usr/include")),
-      ("swift/os", pathsConfiguration.sdkDirPath.appending("usr/include")),
-      ("swift/CoreFoundation", pathsConfiguration.sdkDirPath.appending("usr/include")),
+      ("swift/dispatch", sdkDirPath.appending("usr/include")),
+      ("swift/os", sdkDirPath.appending("usr/include")),
+      ("swift/CoreFoundation", sdkDirPath.appending("usr/include")),
     ] {
       try await rsync(from: distributionPath.appending(pathWithinPackage), to: pathWithinSwiftSDK)
     }

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift

@@ -27,7 +27,10 @@ private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports"
 let byteCountFormatter = ByteCountFormatter()
 
 extension SwiftSDKGenerator {
-  func downloadArtifacts(_ client: HTTPClient, _ engine: Engine) async throws {
+  func downloadArtifacts(
+    _ client: HTTPClient, _ engine: Engine,
+    downloadableArtifacts: inout DownloadableArtifacts
+  ) async throws {
     logGenerationStep("Downloading required toolchain packages...")
     var headRequest = HTTPClientRequest(url: downloadableArtifacts.hostLLVM.remoteURL.absoluteString)
     headRequest.method = .HEAD
@@ -40,7 +43,7 @@ extension SwiftSDKGenerator {
     }
 
     let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in
-      for item in self.downloadableArtifacts.allItems {
+      for item in downloadableArtifacts.allItems {
         group.addTask {
           try await engine[DownloadArtifactQuery(artifact: item)]
         }
@@ -59,18 +62,24 @@ extension SwiftSDKGenerator {
     }
   }
 
-  func downloadUbuntuPackages(_ client: HTTPClient, _ engine: Engine, requiredPackages: [String]) async throws {
+  func downloadUbuntuPackages(
+    _ client: HTTPClient,
+    _ engine: Engine,
+    requiredPackages: [String],
+    versionsConfiguration: VersionsConfiguration,
+    sdkDirPath: FilePath
+  ) async throws {
     logGenerationStep("Parsing Ubuntu packages list...")
 
     async let mainPackages = try await client.parseUbuntuPackagesList(
-      ubuntuRelease: self.versionsConfiguration.linuxDistribution.release,
+      ubuntuRelease: versionsConfiguration.linuxDistribution.release,
       repository: "main",
       targetTriple: self.targetTriple,
       isVerbose: self.isVerbose
     )
 
     async let updatesPackages = try await client.parseUbuntuPackagesList(
-      ubuntuRelease: self.versionsConfiguration.linuxDistribution.release,
+      ubuntuRelease: versionsConfiguration.linuxDistribution.release,
       releaseSuffix: "-updates",
       repository: "main",
       targetTriple: self.targetTriple,
@@ -106,7 +115,7 @@ extension SwiftSDKGenerator {
       report(downloadedFiles: downloadedFiles)
 
       for fileName in urls.map(\.lastPathComponent) {
-        try await fs.unpack(file: tmpDir.appending(fileName), into: pathsConfiguration.sdkDirPath)
+        try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath)
       }
     }
 

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift

@@ -49,19 +49,17 @@ extension SwiftSDKGenerator {
       configuration.httpVersion = .http1Only
       try await withHTTPClient(configuration) { client in
         if !self.isIncremental {
-          try await self.removeRecursively(at: pathsConfiguration.sdkDirPath)
           try await self.removeRecursively(at: pathsConfiguration.toolchainDirPath)
         }
 
         try await self.createDirectoryIfNeeded(at: pathsConfiguration.artifactsCachePath)
-        try await self.createDirectoryIfNeeded(at: pathsConfiguration.sdkDirPath)
         try await self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath)
 
-        try await recipe.makeSwiftSDK(generator: self, engine: engine, httpClient: client)
+        let swiftSDKProduct = try await recipe.makeSwiftSDK(generator: self, engine: engine, httpClient: client)
 
         let toolsetJSONPath = try await generateToolsetJSON(recipe: recipe)
 
-        try await generateDestinationJSON(toolsetPath: toolsetJSONPath)
+        try await generateDestinationJSON(toolsetPath: toolsetJSONPath, sdkDirPath: swiftSDKProduct.sdkDirPath)
 
         try await generateArtifactBundleManifest()
 

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift

@@ -16,10 +16,10 @@ import SystemPackage
 import struct Foundation.Data
 
 extension SwiftSDKGenerator {
-  func fixAbsoluteSymlinks() throws {
+  func fixAbsoluteSymlinks(sdkDirPath: FilePath) throws {
     logGenerationStep("Fixing up absolute symlinks...")
 
-    for (source, absoluteDestination) in try findSymlinks(at: pathsConfiguration.sdkDirPath).filter({
+    for (source, absoluteDestination) in try findSymlinks(at: sdkDirPath).filter({
       $1.string.hasPrefix("/")
     }) {
       guard !absoluteDestination.string.hasPrefix("/etc") else {
@@ -29,7 +29,7 @@ extension SwiftSDKGenerator {
       var relativeSource = source
       var relativeDestination = FilePath()
 
-      let isPrefixRemoved = relativeSource.removePrefix(pathsConfiguration.sdkDirPath)
+      let isPrefixRemoved = relativeSource.removePrefix(sdkDirPath)
       precondition(isPrefixRemoved)
       for _ in relativeSource.removingLastComponent().components {
         relativeDestination.append("..")

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift

@@ -43,13 +43,13 @@ extension SwiftSDKGenerator {
     return toolsetJSONPath
   }
 
-  func generateDestinationJSON(toolsetPath: FilePath) throws {
+  func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath) throws {
     logGenerationStep("Generating destination JSON file...")
 
     let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending("swift-sdk.json")
 
     var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath
-    var relativeSDKDir = pathsConfiguration.sdkDirPath
+    var relativeSDKDir = sdkDirPath
     var relativeToolsetPath = toolsetPath
 
     guard

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift

@@ -32,13 +32,12 @@ private let unusedHostBinaries = [
 ]
 
 extension SwiftSDKGenerator {
-  func unpackHostSwift() async throws {
+  func unpackHostSwift(hostSwiftPackagePath: FilePath) async throws {
     logGenerationStep("Unpacking and copying Swift binaries for the host triple...")
-    let downloadableArtifacts = self.downloadableArtifacts
     let pathsConfiguration = self.pathsConfiguration
 
     try await inTemporaryDirectory { fileSystem, tmpDir in
-      try await fileSystem.unpack(file: downloadableArtifacts.hostSwift.localPath, into: tmpDir)
+      try await fileSystem.unpack(file: hostSwiftPackagePath, into: tmpDir)
       // Remove libraries for platforms we don't intend cross-compiling to
       for platform in unusedDarwinPlatforms {
         try await fileSystem.removeRecursively(at: tmpDir.appending("usr/lib/swift/\(platform)"))
@@ -54,30 +53,22 @@ extension SwiftSDKGenerator {
     }
   }
 
-  func unpackTargetSwiftPackage() async throws {
+  func unpackTargetSwiftPackage(targetSwiftPackagePath: FilePath, relativePathToRoot: [FilePath.Component], sdkDirPath: FilePath) async throws {
     logGenerationStep("Unpacking Swift distribution for the target triple...")
-    let packagePath = downloadableArtifacts.targetSwift.localPath
 
     try await inTemporaryDirectory { fs, tmpDir in
-      try await fs.unpack(file: packagePath, into: tmpDir)
+      try await fs.unpack(file: targetSwiftPackagePath, into: tmpDir)
       try await fs.copyTargetSwift(
-        from: tmpDir.appending(
-          """
-          \(self.versionsConfiguration.swiftDistributionName())/usr/lib
-          """
-        )
+        from: tmpDir.appending(relativePathToRoot).appending("usr/lib"), sdkDirPath: sdkDirPath
       )
     }
   }
 
-  func prepareLLDLinker(_ engine: Engine) async throws {
+  func prepareLLDLinker(_ engine: Engine, llvmArtifact: DownloadableArtifacts.Item) async throws {
     logGenerationStep("Unpacking and copying `lld` linker...")
-    let downloadableArtifacts = self.downloadableArtifacts
     let pathsConfiguration = self.pathsConfiguration
     let targetOS = self.targetTriple.os
 
-    let llvmArtifact = downloadableArtifacts.hostLLVM
-
     let untarDestination = pathsConfiguration.artifactsCachePath.appending(
       FilePath.Component(llvmArtifact.localPath.stem!)!.stem
     )
@@ -104,7 +95,7 @@ extension SwiftSDKGenerator {
     case .wasi:
       pathsConfiguration.toolchainBinDirPath.appending("wasm-ld")
     default:
-      fatalError()
+      fatalError("Unknown target OS to prepare lld \"\(targetOS)\"")
     }
 
     try self.copy(from: unpackedLLDPath, to: toolchainLLDPath)

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift

@@ -22,27 +22,17 @@ public actor SwiftSDKGenerator {
   let hostTriple: Triple
   let targetTriple: Triple
   let artifactID: String
-  let versionsConfiguration: VersionsConfiguration
   let pathsConfiguration: PathsConfiguration
-  var downloadableArtifacts: DownloadableArtifacts
-  let shouldUseDocker: Bool
-  let baseDockerImage: String?
   let isIncremental: Bool
   let isVerbose: Bool
   let engineCachePath: SQLite.Location
   let logger: Logger
 
   public init(
     bundleVersion: String,
-    hostCPUArchitecture: Triple.CPU?,
-    targetCPUArchitecture: Triple.CPU?,
-    swiftVersion: String,
-    swiftBranch: String?,
-    lldVersion: String,
-    linuxDistribution: LinuxDistribution,
-    shouldUseDocker: Bool,
-    baseDockerImage: String?,
-    artifactID: String?,
+    hostTriple: Triple,
+    targetTriple: Triple,
+    artifactID: String,
     isIncremental: Bool,
     isVerbose: Bool,
     logger: Logger
@@ -56,52 +46,16 @@ public actor SwiftSDKGenerator {
       .removingLastComponent()
 
     self.bundleVersion = bundleVersion
+    self.hostTriple = hostTriple
 
-    var currentTriple = try await Self.getCurrentTriple(isVerbose: isVerbose)
-    if let hostCPUArchitecture {
-      currentTriple.cpu = hostCPUArchitecture
-    }
-
-    self.hostTriple = currentTriple
+    self.targetTriple = targetTriple
+    self.artifactID = artifactID
 
-    self.targetTriple = Triple(
-      cpu: targetCPUArchitecture ?? self.hostTriple.cpu,
-      vendor: .unknown,
-      os: .linux,
-      environment: .gnu
-    )
-    self.artifactID = artifactID ?? """
-    \(swiftVersion)_\(linuxDistribution.name.rawValue)_\(linuxDistribution.release)_\(
-      self.targetTriple.cpu.linuxConventionName
-    )
-    """
-
-    self.versionsConfiguration = try .init(
-      swiftVersion: swiftVersion,
-      swiftBranch: swiftBranch,
-      lldVersion: lldVersion,
-      linuxDistribution: linuxDistribution,
-      targetTriple: self.targetTriple
-    )
     self.pathsConfiguration = .init(
       sourceRoot: sourceRoot,
       artifactID: self.artifactID,
-      linuxDistribution: self.versionsConfiguration.linuxDistribution,
       targetTriple: self.targetTriple
     )
-    self.downloadableArtifacts = try .init(
-      hostTriple: self.hostTriple,
-      targetTriple: self.targetTriple,
-      shouldUseDocker: shouldUseDocker,
-      self.versionsConfiguration,
-      self.pathsConfiguration
-    )
-    self.shouldUseDocker = shouldUseDocker
-    self.baseDockerImage = if shouldUseDocker {
-      baseDockerImage ?? self.versionsConfiguration.swiftBaseDockerImage
-    } else {
-      nil
-    }
     self.isIncremental = isIncremental
     self.isVerbose = isVerbose
 
@@ -112,6 +66,14 @@ public actor SwiftSDKGenerator {
   private let fileManager = FileManager.default
   private static let dockerCommand = "docker"
 
+  public static func getHostTriple(explicitArch hostCPUArchitecture: Triple.CPU?, isVerbose: Bool) async throws -> Triple {
+    var currentTriple = try await Self.getCurrentTriple(isVerbose: isVerbose)
+    if let hostCPUArchitecture {
+      currentTriple.cpu = hostCPUArchitecture
+    }
+    return currentTriple
+  }
+
   static func getCurrentTriple(isVerbose: Bool) async throws -> Triple {
     let cpuString = try await Shell.readStdout("uname -m", shouldLogCommands: isVerbose)
       .trimmingCharacters(in: .whitespacesAndNewlines)