From 95c642d0ea6a7685f665f2d5866864f0afeb1fcd Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 15 Jan 2024 08:00:31 +0900 Subject: [PATCH 1/4] Recipe: Move minor Linux-specific variables into recipe This is a preparation for moving `linuxDistribution` into recipe. --- Sources/GeneratorCLI/GeneratorCLI.swift | 14 +- .../Generator/SwiftSDKGenerator+Copy.swift | 26 ++-- .../SwiftSDKGenerator+Download.swift | 21 ++- .../SwiftSDKGenerator+Entrypoint.swift | 6 +- .../Generator/SwiftSDKGenerator+Fixup.swift | 6 +- .../SwiftSDKGenerator+Metadata.swift | 4 +- .../Generator/SwiftSDKGenerator+Unpack.swift | 23 +-- .../Generator/SwiftSDKGenerator.swift | 32 +---- .../PathsConfiguration.swift | 5 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 132 ++++++++++++++++-- .../SwiftSDKRecipes/SwiftSDKRecipe.swift | 7 +- .../ArchitectureMappingTests.swift | 23 ++- 12 files changed, 194 insertions(+), 105 deletions(-) diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index 03154a7..4d2e7d1 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -110,17 +110,21 @@ struct GeneratorCLI: AsyncParsableCommand { hostCPUArchitecture: self.hostArch, targetCPUArchitecture: self.targetArch, swiftVersion: self.swiftVersion, - swiftBranch: self.swiftBranch, - lldVersion: self.lldVersion, linuxDistribution: linuxDistribution, - shouldUseDocker: self.withDocker, - baseDockerImage: self.fromContainerImage, artifactID: self.sdkName, isIncremental: self.incremental, isVerbose: self.verbose, logger: logger ) - let recipe = LinuxRecipe() + let recipe = try await LinuxRecipe( + targetTriple: generator.targetTriple, + linuxDistribution: linuxDistribution, + swiftVersion: swiftVersion, + swiftBranch: swiftBranch, + lldVersion: lldVersion, + withDocker: withDocker, + fromContainerImage: fromContainerImage + ) let serviceGroup = ServiceGroup( configuration: .init( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 0afe270..aa73146 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -13,13 +13,11 @@ import SystemPackage extension SwiftSDKGenerator { - func copyTargetSwiftFromDocker() async throws { + func copyTargetSwiftFromDocker(targetDistribution: LinuxDistribution, baseDockerImage: String, sdkDirPath: FilePath) async throws { logGenerationStep("Launching a Docker container to copy Swift SDK for the target triple from it...") - try await withDockerContainer(fromImage: baseDockerImage!) { containerID in - let pathsConfiguration = self.pathsConfiguration - + try await withDockerContainer(fromImage: baseDockerImage) { containerID in try await inTemporaryDirectory { generator, _ in - let sdkUsrPath = pathsConfiguration.sdkDirPath.appending("usr") + let sdkUsrPath = sdkDirPath.appending("usr") let sdkUsrLibPath = sdkUsrPath.appending("lib") try await generator.createDirectoryIfNeeded(at: sdkUsrPath) try await generator.copyFromDockerContainer( @@ -28,7 +26,7 @@ extension SwiftSDKGenerator { to: sdkUsrPath.appending("include") ) - if case .rhel = self.versionsConfiguration.linuxDistribution { + if case .rhel = targetDistribution { try await generator.runOnDockerContainer( id: containerID, command: #""" @@ -56,7 +54,7 @@ extension SwiftSDKGenerator { from: containerLib64, to: sdkUsrLib64Path ) - try await createSymlink(at: pathsConfiguration.sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") + try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") } try await generator.createDirectoryIfNeeded(at: sdkUsrLibPath) @@ -68,7 +66,7 @@ extension SwiftSDKGenerator { // architecture-specific directories: // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. - if case .ubuntu = self.versionsConfiguration.linuxDistribution { + if case .ubuntu = targetDistribution { subpaths += [("\(targetTriple.cpu)-linux-gnu", false)] } @@ -80,27 +78,27 @@ extension SwiftSDKGenerator { failIfNotExists: failIfNotExists ) } - try await generator.createSymlink(at: pathsConfiguration.sdkDirPath.appending("lib"), pointingTo: "usr/lib") + try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib") // Python artifacts are redundant. try await generator.removeRecursively(at: sdkUsrLibPath.appending("python3.10")) try await generator.removeRecursively(at: sdkUsrLibPath.appending("ssl")) - try await generator.copyTargetSwift(from: sdkUsrLibPath) + try await generator.copyTargetSwift(from: sdkUsrLibPath, sdkDirPath: sdkDirPath) } } } - func copyTargetSwift(from distributionPath: FilePath) async throws { + func copyTargetSwift(from distributionPath: FilePath, sdkDirPath: FilePath) async throws { logGenerationStep("Copying Swift core libraries for the target triple into Swift SDK bundle...") for (pathWithinPackage, pathWithinSwiftSDK) in [ ("swift/linux", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift")), ("swift_static/linux", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static")), ("swift_static/shims", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static")), - ("swift/dispatch", pathsConfiguration.sdkDirPath.appending("usr/include")), - ("swift/os", pathsConfiguration.sdkDirPath.appending("usr/include")), - ("swift/CoreFoundation", pathsConfiguration.sdkDirPath.appending("usr/include")), + ("swift/dispatch", sdkDirPath.appending("usr/include")), + ("swift/os", sdkDirPath.appending("usr/include")), + ("swift/CoreFoundation", sdkDirPath.appending("usr/include")), ] { try await rsync(from: distributionPath.appending(pathWithinPackage), to: pathWithinSwiftSDK) } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index f35a958..8117e81 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -27,7 +27,10 @@ private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" let byteCountFormatter = ByteCountFormatter() extension SwiftSDKGenerator { - func downloadArtifacts(_ client: HTTPClient, _ engine: Engine) async throws { + func downloadArtifacts( + _ client: HTTPClient, _ engine: Engine, + downloadableArtifacts: inout DownloadableArtifacts + ) async throws { logGenerationStep("Downloading required toolchain packages...") var headRequest = HTTPClientRequest(url: downloadableArtifacts.hostLLVM.remoteURL.absoluteString) headRequest.method = .HEAD @@ -40,7 +43,7 @@ extension SwiftSDKGenerator { } let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in - for item in self.downloadableArtifacts.allItems { + for item in downloadableArtifacts.allItems { group.addTask { try await engine[DownloadArtifactQuery(artifact: item)] } @@ -59,18 +62,24 @@ extension SwiftSDKGenerator { } } - func downloadUbuntuPackages(_ client: HTTPClient, _ engine: Engine, requiredPackages: [String]) async throws { + func downloadUbuntuPackages( + _ client: HTTPClient, + _ engine: Engine, + requiredPackages: [String], + versionsConfiguration: VersionsConfiguration, + sdkDirPath: FilePath + ) async throws { logGenerationStep("Parsing Ubuntu packages list...") async let mainPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: self.versionsConfiguration.linuxDistribution.release, + ubuntuRelease: versionsConfiguration.linuxDistribution.release, repository: "main", targetTriple: self.targetTriple, isVerbose: self.isVerbose ) async let updatesPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: self.versionsConfiguration.linuxDistribution.release, + ubuntuRelease: versionsConfiguration.linuxDistribution.release, releaseSuffix: "-updates", repository: "main", targetTriple: self.targetTriple, @@ -106,7 +115,7 @@ extension SwiftSDKGenerator { report(downloadedFiles: downloadedFiles) for fileName in urls.map(\.lastPathComponent) { - try await fs.unpack(file: tmpDir.appending(fileName), into: pathsConfiguration.sdkDirPath) + try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath) } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index 4a684be..7ef9a35 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -49,19 +49,17 @@ extension SwiftSDKGenerator { configuration.httpVersion = .http1Only try await withHTTPClient(configuration) { client in if !self.isIncremental { - try await self.removeRecursively(at: pathsConfiguration.sdkDirPath) try await self.removeRecursively(at: pathsConfiguration.toolchainDirPath) } try await self.createDirectoryIfNeeded(at: pathsConfiguration.artifactsCachePath) - try await self.createDirectoryIfNeeded(at: pathsConfiguration.sdkDirPath) try await self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath) - try await recipe.makeSwiftSDK(generator: self, engine: engine, httpClient: client) + let swiftSdkProduct = try await recipe.makeSwiftSDK(generator: self, engine: engine, httpClient: client) let toolsetJSONPath = try await generateToolsetJSON(recipe: recipe) - try await generateDestinationJSON(toolsetPath: toolsetJSONPath) + try await generateDestinationJSON(toolsetPath: toolsetJSONPath, sdkDirPath: swiftSdkProduct.sdkDirPath) try await generateArtifactBundleManifest() diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift index b421a67..807e513 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift @@ -16,10 +16,10 @@ import SystemPackage import struct Foundation.Data extension SwiftSDKGenerator { - func fixAbsoluteSymlinks() throws { + func fixAbsoluteSymlinks(sdkDirPath: FilePath) throws { logGenerationStep("Fixing up absolute symlinks...") - for (source, absoluteDestination) in try findSymlinks(at: pathsConfiguration.sdkDirPath).filter({ + for (source, absoluteDestination) in try findSymlinks(at: sdkDirPath).filter({ $1.string.hasPrefix("/") }) { guard !absoluteDestination.string.hasPrefix("/etc") else { @@ -29,7 +29,7 @@ extension SwiftSDKGenerator { var relativeSource = source var relativeDestination = FilePath() - let isPrefixRemoved = relativeSource.removePrefix(pathsConfiguration.sdkDirPath) + let isPrefixRemoved = relativeSource.removePrefix(sdkDirPath) precondition(isPrefixRemoved) for _ in relativeSource.removingLastComponent().components { relativeDestination.append("..") diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index a662e36..95c58f7 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -43,13 +43,13 @@ extension SwiftSDKGenerator { return toolsetJSONPath } - func generateDestinationJSON(toolsetPath: FilePath) throws { + func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath) throws { logGenerationStep("Generating destination JSON file...") let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending("swift-sdk.json") var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath - var relativeSDKDir = pathsConfiguration.sdkDirPath + var relativeSDKDir = sdkDirPath var relativeToolsetPath = toolsetPath guard diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index 8993ebf..4c2d476 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -32,13 +32,12 @@ private let unusedHostBinaries = [ ] extension SwiftSDKGenerator { - func unpackHostSwift() async throws { + func unpackHostSwift(hostSwiftPackagePath: FilePath) async throws { logGenerationStep("Unpacking and copying Swift binaries for the host triple...") - let downloadableArtifacts = self.downloadableArtifacts let pathsConfiguration = self.pathsConfiguration try await inTemporaryDirectory { fileSystem, tmpDir in - try await fileSystem.unpack(file: downloadableArtifacts.hostSwift.localPath, into: tmpDir) + try await fileSystem.unpack(file: hostSwiftPackagePath, into: tmpDir) // Remove libraries for platforms we don't intend cross-compiling to for platform in unusedDarwinPlatforms { try await fileSystem.removeRecursively(at: tmpDir.appending("usr/lib/swift/\(platform)")) @@ -54,30 +53,22 @@ extension SwiftSDKGenerator { } } - func unpackTargetSwiftPackage() async throws { + func unpackTargetSwiftPackage(targetSwiftPackagePath: FilePath, relativePathToRoot: [FilePath.Component], sdkDirPath: FilePath) async throws { logGenerationStep("Unpacking Swift distribution for the target triple...") - let packagePath = downloadableArtifacts.targetSwift.localPath try await inTemporaryDirectory { fs, tmpDir in - try await fs.unpack(file: packagePath, into: tmpDir) + try await fs.unpack(file: targetSwiftPackagePath, into: tmpDir) try await fs.copyTargetSwift( - from: tmpDir.appending( - """ - \(self.versionsConfiguration.swiftDistributionName())/usr/lib - """ - ) + from: tmpDir.appending(relativePathToRoot).appending("usr/lib"), sdkDirPath: sdkDirPath ) } } - func prepareLLDLinker(_ engine: Engine) async throws { + func prepareLLDLinker(_ engine: Engine, llvmArtifact: DownloadableArtifacts.Item) async throws { logGenerationStep("Unpacking and copying `lld` linker...") - let downloadableArtifacts = self.downloadableArtifacts let pathsConfiguration = self.pathsConfiguration let targetOS = self.targetTriple.os - let llvmArtifact = downloadableArtifacts.hostLLVM - let untarDestination = pathsConfiguration.artifactsCachePath.appending( FilePath.Component(llvmArtifact.localPath.stem!)!.stem ) @@ -104,7 +95,7 @@ extension SwiftSDKGenerator { case .wasi: pathsConfiguration.toolchainBinDirPath.appending("wasm-ld") default: - fatalError() + fatalError("Unknown target OS to prepare lld \"\(targetOS)\"") } try self.copy(from: unpackedLLDPath, to: toolchainLLDPath) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index 9888a9f..b2fab8a 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -20,13 +20,10 @@ import Helpers public actor SwiftSDKGenerator { let bundleVersion: String let hostTriple: Triple - let targetTriple: Triple + // FIXME: This is temporaliry public while refactoring + public let targetTriple: Triple let artifactID: String - let versionsConfiguration: VersionsConfiguration let pathsConfiguration: PathsConfiguration - var downloadableArtifacts: DownloadableArtifacts - let shouldUseDocker: Bool - let baseDockerImage: String? let isIncremental: Bool let isVerbose: Bool let engineCachePath: SQLite.Location @@ -37,11 +34,7 @@ public actor SwiftSDKGenerator { hostCPUArchitecture: Triple.CPU?, targetCPUArchitecture: Triple.CPU?, swiftVersion: String, - swiftBranch: String?, - lldVersion: String, linuxDistribution: LinuxDistribution, - shouldUseDocker: Bool, - baseDockerImage: String?, artifactID: String?, isIncremental: Bool, isVerbose: Bool, @@ -76,32 +69,11 @@ public actor SwiftSDKGenerator { ) """ - self.versionsConfiguration = try .init( - swiftVersion: swiftVersion, - swiftBranch: swiftBranch, - lldVersion: lldVersion, - linuxDistribution: linuxDistribution, - targetTriple: self.targetTriple - ) self.pathsConfiguration = .init( sourceRoot: sourceRoot, artifactID: self.artifactID, - linuxDistribution: self.versionsConfiguration.linuxDistribution, targetTriple: self.targetTriple ) - self.downloadableArtifacts = try .init( - hostTriple: self.hostTriple, - targetTriple: self.targetTriple, - shouldUseDocker: shouldUseDocker, - self.versionsConfiguration, - self.pathsConfiguration - ) - self.shouldUseDocker = shouldUseDocker - self.baseDockerImage = if shouldUseDocker { - baseDockerImage ?? self.versionsConfiguration.swiftBaseDockerImage - } else { - nil - } self.isIncremental = isIncremental self.isVerbose = isVerbose diff --git a/Sources/SwiftSDKGenerator/PathsConfiguration.swift b/Sources/SwiftSDKGenerator/PathsConfiguration.swift index 5d2585e..1f64f28 100644 --- a/Sources/SwiftSDKGenerator/PathsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PathsConfiguration.swift @@ -13,7 +13,7 @@ import struct SystemPackage.FilePath public struct PathsConfiguration: Sendable { - init(sourceRoot: FilePath, artifactID: String, linuxDistribution: LinuxDistribution, targetTriple: Triple) { + init(sourceRoot: FilePath, artifactID: String, targetTriple: Triple) { self.sourceRoot = sourceRoot self.artifactBundlePath = sourceRoot .appending("Bundles") @@ -22,8 +22,6 @@ public struct PathsConfiguration: Sendable { self.swiftSDKRootPath = self.artifactBundlePath .appending(artifactID) .appending(targetTriple.linuxConventionDescription) - self.sdkDirPath = self.swiftSDKRootPath - .appending("\(linuxDistribution.name.rawValue)-\(linuxDistribution.release).sdk") self.toolchainDirPath = self.swiftSDKRootPath.appending("swift.xctoolchain") self.toolchainBinDirPath = self.toolchainDirPath.appending("usr/bin") } @@ -32,7 +30,6 @@ public struct PathsConfiguration: Sendable { let artifactBundlePath: FilePath let artifactsCachePath: FilePath let swiftSDKRootPath: FilePath - let sdkDirPath: FilePath let toolchainDirPath: FilePath let toolchainBinDirPath: FilePath } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index bd29c91..914d0ff 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -11,10 +11,72 @@ //===----------------------------------------------------------------------===// import AsyncHTTPClient +import Foundation import GeneratorEngine +import struct SystemPackage.FilePath public struct LinuxRecipe: SwiftSDKRecipe { - public init() {} + public enum TargetSwiftSource: Sendable { + case docker(baseSwiftDockerImage: String) + case tarball + } + + let mainTargetTriple: Triple + let linuxDistribution: LinuxDistribution + let targetSwiftSource: TargetSwiftSource + let versionsConfiguration: VersionsConfiguration + + var shouldUseDocker: Bool { + if case .docker = self.targetSwiftSource { + return true + } + return false + } + + public init( + targetTriple: Triple, + linuxDistribution: LinuxDistribution, + swiftVersion: String, + swiftBranch: String?, + lldVersion: String, + withDocker: Bool, + fromContainerImage: String? + ) throws { + let versionsConfiguration = try VersionsConfiguration( + swiftVersion: swiftVersion, + swiftBranch: swiftBranch, + lldVersion: lldVersion, + linuxDistribution: linuxDistribution, + targetTriple: targetTriple + ) + + let targetSwiftSource: LinuxRecipe.TargetSwiftSource + if withDocker { + let imageName = fromContainerImage ?? versionsConfiguration.swiftBaseDockerImage + targetSwiftSource = .docker(baseSwiftDockerImage: imageName) + } else { + targetSwiftSource = .tarball + } + + self.init( + mainTargetTriple: targetTriple, + linuxDistribution: linuxDistribution, + targetSwiftSource: targetSwiftSource, + versionsConfiguration: versionsConfiguration + ) + } + + public init( + mainTargetTriple: Triple, + linuxDistribution: LinuxDistribution, + targetSwiftSource: TargetSwiftSource, + versionsConfiguration: VersionsConfiguration + ) { + self.mainTargetTriple = mainTargetTriple + self.linuxDistribution = linuxDistribution + self.targetSwiftSource = targetSwiftSource + self.versionsConfiguration = versionsConfiguration + } public func applyPlatformOptions(toolset: inout Toolset) { toolset.swiftCompiler = Toolset.ToolProperties(extraCLIOptions: ["-use-ld=lld", "-Xlinker", "-R/usr/lib/swift/linux/"]) @@ -23,29 +85,69 @@ public struct LinuxRecipe: SwiftSDKRecipe { toolset.librarian = Toolset.ToolProperties(path: "llvm-ar") } - public func makeSwiftSDK(generator: SwiftSDKGenerator, engine: Engine, httpClient client: HTTPClient) async throws { - try await generator.downloadArtifacts(client, engine) + func sdkDirPath(paths: PathsConfiguration) -> FilePath { + paths.swiftSDKRootPath + .appending("\(linuxDistribution.name.rawValue)-\(linuxDistribution.release).sdk") + } + + public func makeSwiftSDK( + generator: SwiftSDKGenerator, + engine: Engine, + httpClient client: HTTPClient + ) async throws -> SwiftSDKProduct { + let sdkDirPath = self.sdkDirPath(paths: generator.pathsConfiguration) + if generator.isIncremental { + try await generator.removeRecursively(at: sdkDirPath) + } + try await generator.createDirectoryIfNeeded(at: sdkDirPath) + + var downloadableArtifacts = try DownloadableArtifacts( + hostTriple: generator.hostTriple, + targetTriple: generator.targetTriple, + shouldUseDocker: shouldUseDocker, + versionsConfiguration, + generator.pathsConfiguration + ) + + try await generator.downloadArtifacts(client, engine, downloadableArtifacts: &downloadableArtifacts) - if !generator.shouldUseDocker { - guard case let .ubuntu(version) = generator.versionsConfiguration.linuxDistribution else { + if !self.shouldUseDocker { + guard case let .ubuntu(version) = linuxDistribution else { throw GeneratorError - .distributionSupportsOnlyDockerGenerator(generator.versionsConfiguration.linuxDistribution) + .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) } - try await generator.downloadUbuntuPackages(client, engine, requiredPackages: version.requiredPackages) + try await generator.downloadUbuntuPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: versionsConfiguration, + sdkDirPath: sdkDirPath + ) } - try await generator.unpackHostSwift() + try await generator.unpackHostSwift( + hostSwiftPackagePath: downloadableArtifacts.hostSwift.localPath + ) - if generator.shouldUseDocker { - try await generator.copyTargetSwiftFromDocker() - } else { - try await generator.unpackTargetSwiftPackage() + switch self.targetSwiftSource { + case let .docker(baseSwiftDockerImage): + try await generator.copyTargetSwiftFromDocker( + targetDistribution: self.linuxDistribution, + baseDockerImage: baseSwiftDockerImage, + sdkDirPath: sdkDirPath + ) + case .tarball: + try await generator.unpackTargetSwiftPackage( + targetSwiftPackagePath: downloadableArtifacts.targetSwift.localPath, + relativePathToRoot: [FilePath.Component(versionsConfiguration.swiftDistributionName())!], + sdkDirPath: sdkDirPath + ) } - try await generator.prepareLLDLinker(engine) + try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM) - try await generator.fixAbsoluteSymlinks() + try await generator.fixAbsoluteSymlinks(sdkDirPath: sdkDirPath) let targetCPU = generator.targetTriple.cpu try await generator.fixGlibcModuleMap( @@ -61,5 +163,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { logGenerationStep("Fixing `swift-autolink-extract` symlink...") try await generator.createSymlink(at: autolinkExtractPath, pointingTo: "swift") } + + return SwiftSDKProduct(sdkDirPath: sdkDirPath) } } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 7891129..0964012 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -12,6 +12,11 @@ import AsyncHTTPClient import GeneratorEngine +import struct SystemPackage.FilePath + +public struct SwiftSDKProduct { + let sdkDirPath: FilePath +} /// A protocol describing a set of platform specific instructions to make a Swift SDK public protocol SwiftSDKRecipe: Sendable { @@ -19,7 +24,7 @@ public protocol SwiftSDKRecipe: Sendable { func applyPlatformOptions(toolset: inout Toolset) /// The main entrypoint of the recipe to make a Swift SDK - func makeSwiftSDK(generator: SwiftSDKGenerator, engine: Engine, httpClient: HTTPClient) async throws + func makeSwiftSDK(generator: SwiftSDKGenerator, engine: Engine, httpClient: HTTPClient) async throws -> SwiftSDKProduct } extension SwiftSDKRecipe { diff --git a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift index 8b1ac39..04f5e23 100644 --- a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift +++ b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift @@ -56,22 +56,33 @@ final class ArchitectureMappingTests: XCTestCase { // Remaining fields are placeholders which are the same for all // combinations of build and runtime architecture swiftVersion: "5.8-RELEASE", - swiftBranch: nil, - lldVersion: "16.0.4", linuxDistribution: .ubuntu(.jammy), - shouldUseDocker: false, - baseDockerImage: nil, artifactID: nil, isIncremental: false, isVerbose: false, logger: Logger(label: "org.swift.swift-sdk-generator") ) + let recipe = try await LinuxRecipe( + targetTriple: sdk.targetTriple, + linuxDistribution: .ubuntu(.jammy), + swiftVersion: "5.8-RELEASE", + swiftBranch: nil, + lldVersion: "16.0.4", + withDocker: false, + fromContainerImage: nil + ) let sdkArtifactID = await sdk.artifactID XCTAssertEqual(sdkArtifactID, artifactID, "Unexpected artifactID") // Verify download URLs - let artifacts = await sdk.downloadableArtifacts + let artifacts = try await DownloadableArtifacts( + hostTriple: sdk.hostTriple, + targetTriple: sdk.targetTriple, + shouldUseDocker: recipe.shouldUseDocker, + recipe.versionsConfiguration, + sdk.pathsConfiguration + ) // The build-time Swift SDK is a multiarch package and so is always the same XCTAssertEqual( @@ -107,7 +118,7 @@ final class ArchitectureMappingTests: XCTestCase { // The SDK path must use Swift's name for the architecture XCTAssertEqual( - paths.sdkDirPath.string, + recipe.sdkDirPath(paths: paths).string, paths.artifactBundlePath.string + sdkDirPathSuffix, "Unexpected sdkDirPathSuffix" ) From 9f5b6b4aad56606c1c1f7f40ad857bb6e09c0b85 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 15 Jan 2024 16:21:15 +0900 Subject: [PATCH 2/4] Make `Triple`'s initializer public --- .../PlatformModels/Triple.swift | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift b/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift index 062df36..541a203 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift @@ -49,19 +49,19 @@ public struct Triple: Sendable, CustomStringConvertible { } } - enum Vendor: String { + public enum Vendor: String, Sendable { case apple case unknown } - enum OS: Hashable, CustomStringConvertible { + public enum OS: Hashable, CustomStringConvertible, Sendable { case linux case darwin(version: String) case macosx(version: String) case wasi case win32 - var description: String { + public var description: String { switch self { case .linux: "linux" @@ -77,14 +77,21 @@ public struct Triple: Sendable, CustomStringConvertible { } } - enum Environment { + public enum Environment: Sendable { case gnu } - var cpu: CPU - var vendor: Vendor - var os: OS - var environment: Environment? + public var cpu: CPU + public var vendor: Vendor + public var os: OS + public var environment: Environment? + + public init(cpu: CPU, vendor: Vendor, os: OS, environment: Environment? = nil) { + self.cpu = cpu + self.vendor = vendor + self.os = os + self.environment = environment + } public var linuxConventionDescription: String { "\(self.cpu.linuxConventionName)-\(self.vendor)-\(self.os)\(self.environment != nil ? "-\(self.environment!)" : "")" From 58b81819a6a1aa6ef210bb1032ccd4fb1525f512 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 15 Jan 2024 16:22:06 +0900 Subject: [PATCH 3/4] Recipe: Get rid of all use of `LinuxDistribution` in SDKGenerator --- Sources/GeneratorCLI/GeneratorCLI.swift | 29 ++++++++------ .../Generator/SwiftSDKGenerator.swift | 40 +++++++------------ .../SwiftSDKRecipes/LinuxRecipe.swift | 8 ++++ .../SwiftSDKRecipes/SwiftSDKRecipe.swift | 3 ++ .../ArchitectureMappingTests.swift | 28 ++++++------- 5 files changed, 56 insertions(+), 52 deletions(-) diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index 4d2e7d1..b49b92b 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -105,19 +105,15 @@ struct GeneratorCLI: AsyncParsableCommand { let elapsed = try await ContinuousClock().measure { let logger = Logger(label: "org.swift.swift-sdk-generator") - let generator = try await SwiftSDKGenerator( - bundleVersion: self.bundleVersion, - hostCPUArchitecture: self.hostArch, - targetCPUArchitecture: self.targetArch, - swiftVersion: self.swiftVersion, - linuxDistribution: linuxDistribution, - artifactID: self.sdkName, - isIncremental: self.incremental, - isVerbose: self.verbose, - logger: logger + let hostTriple = try await SwiftSDKGenerator.getHostTriple(explicitArch: hostArch, isVerbose: verbose) + let targetTriple = Triple( + cpu: targetArch ?? hostTriple.cpu, + vendor: .unknown, + os: .linux, + environment: .gnu ) - let recipe = try await LinuxRecipe( - targetTriple: generator.targetTriple, + let recipe = try LinuxRecipe( + targetTriple: targetTriple, linuxDistribution: linuxDistribution, swiftVersion: swiftVersion, swiftBranch: swiftBranch, @@ -125,6 +121,15 @@ struct GeneratorCLI: AsyncParsableCommand { withDocker: withDocker, fromContainerImage: fromContainerImage ) + let generator = try await SwiftSDKGenerator( + bundleVersion: self.bundleVersion, + hostTriple: hostTriple, + targetTriple: targetTriple, + artifactID: self.sdkName ?? recipe.defaultArtifactID(), + isIncremental: self.incremental, + isVerbose: self.verbose, + logger: logger + ) let serviceGroup = ServiceGroup( configuration: .init( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index b2fab8a..600f871 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -20,8 +20,7 @@ import Helpers public actor SwiftSDKGenerator { let bundleVersion: String let hostTriple: Triple - // FIXME: This is temporaliry public while refactoring - public let targetTriple: Triple + let targetTriple: Triple let artifactID: String let pathsConfiguration: PathsConfiguration let isIncremental: Bool @@ -31,11 +30,9 @@ public actor SwiftSDKGenerator { public init( bundleVersion: String, - hostCPUArchitecture: Triple.CPU?, - targetCPUArchitecture: Triple.CPU?, - swiftVersion: String, - linuxDistribution: LinuxDistribution, - artifactID: String?, + hostTriple: Triple, + targetTriple: Triple, + artifactID: String, isIncremental: Bool, isVerbose: Bool, logger: Logger @@ -49,25 +46,10 @@ public actor SwiftSDKGenerator { .removingLastComponent() self.bundleVersion = bundleVersion + self.hostTriple = hostTriple - var currentTriple = try await Self.getCurrentTriple(isVerbose: isVerbose) - if let hostCPUArchitecture { - currentTriple.cpu = hostCPUArchitecture - } - - self.hostTriple = currentTriple - - self.targetTriple = Triple( - cpu: targetCPUArchitecture ?? self.hostTriple.cpu, - vendor: .unknown, - os: .linux, - environment: .gnu - ) - self.artifactID = artifactID ?? """ - \(swiftVersion)_\(linuxDistribution.name.rawValue)_\(linuxDistribution.release)_\( - self.targetTriple.cpu.linuxConventionName - ) - """ + self.targetTriple = targetTriple + self.artifactID = artifactID self.pathsConfiguration = .init( sourceRoot: sourceRoot, @@ -84,6 +66,14 @@ public actor SwiftSDKGenerator { private let fileManager = FileManager.default private static let dockerCommand = "docker" + public static func getHostTriple(explicitArch hostCPUArchitecture: Triple.CPU?, isVerbose: Bool) async throws -> Triple { + var currentTriple = try await Self.getCurrentTriple(isVerbose: isVerbose) + if let hostCPUArchitecture { + currentTriple.cpu = hostCPUArchitecture + } + return currentTriple + } + static func getCurrentTriple(isVerbose: Bool) async throws -> Triple { let cpuString = try await Shell.readStdout("uname -m", shouldLogCommands: isVerbose) .trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 914d0ff..58b0bde 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -85,6 +85,14 @@ public struct LinuxRecipe: SwiftSDKRecipe { toolset.librarian = Toolset.ToolProperties(path: "llvm-ar") } + public func defaultArtifactID() -> String { + """ + \(versionsConfiguration.swiftVersion)_\(linuxDistribution.name.rawValue)_\(linuxDistribution.release)_\( + mainTargetTriple.cpu.linuxConventionName + ) + """ + } + func sdkDirPath(paths: PathsConfiguration) -> FilePath { paths.swiftSDKRootPath .appending("\(linuxDistribution.name.rawValue)-\(linuxDistribution.release).sdk") diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 0964012..84892e9 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -23,6 +23,9 @@ public protocol SwiftSDKRecipe: Sendable { /// Update the given toolset with platform specific options func applyPlatformOptions(toolset: inout Toolset) + /// Returns the default identifier of the Swift SDK + func defaultArtifactID() -> String + /// The main entrypoint of the recipe to make a Swift SDK func makeSwiftSDK(generator: SwiftSDKGenerator, engine: Engine, httpClient: HTTPClient) async throws -> SwiftSDKProduct } diff --git a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift index 04f5e23..c3db889 100644 --- a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift +++ b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift @@ -44,33 +44,31 @@ final class ArchitectureMappingTests: XCTestCase { artifactBundlePathSuffix: String, // Path to the generated bundle sdkDirPathSuffix: String // Path of the SDK within the bundle ) async throws { + let targetTriple = Triple(cpu: targetCPUArchitecture, vendor: .unknown, os: .linux, environment: .gnu) + let recipe = try LinuxRecipe( + targetTriple: targetTriple, + linuxDistribution: .ubuntu(.jammy), + swiftVersion: "5.8-RELEASE", + swiftBranch: nil, + lldVersion: "16.0.4", + withDocker: false, + fromContainerImage: nil + ) // LocalSwiftSDKGenerator constructs URLs and paths which depend on architectures let sdk = try await SwiftSDKGenerator( bundleVersion: bundleVersion, // macOS is currently the only supported build environment - hostCPUArchitecture: hostCPUArchitecture, + hostTriple: Triple(cpu: hostCPUArchitecture, vendor: .apple, os: .macosx(version: "13")), // Linux is currently the only supported runtime environment - targetCPUArchitecture: targetCPUArchitecture, - + targetTriple: targetTriple, + artifactID: recipe.defaultArtifactID(), // Remaining fields are placeholders which are the same for all // combinations of build and runtime architecture - swiftVersion: "5.8-RELEASE", - linuxDistribution: .ubuntu(.jammy), - artifactID: nil, isIncremental: false, isVerbose: false, logger: Logger(label: "org.swift.swift-sdk-generator") ) - let recipe = try await LinuxRecipe( - targetTriple: sdk.targetTriple, - linuxDistribution: .ubuntu(.jammy), - swiftVersion: "5.8-RELEASE", - swiftBranch: nil, - lldVersion: "16.0.4", - withDocker: false, - fromContainerImage: nil - ) let sdkArtifactID = await sdk.artifactID XCTAssertEqual(sdkArtifactID, artifactID, "Unexpected artifactID") From 9f1a836c539b7f0bff78a75fe3031c5081dced05 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 15 Jan 2024 23:52:32 +0900 Subject: [PATCH 4/4] Minor review feedbacks --- Sources/GeneratorCLI/GeneratorCLI.swift | 2 +- .../Generator/SwiftSDKGenerator+Entrypoint.swift | 4 ++-- Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift | 2 +- .../SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift | 4 ++-- Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index b49b92b..015d342 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -125,7 +125,7 @@ struct GeneratorCLI: AsyncParsableCommand { bundleVersion: self.bundleVersion, hostTriple: hostTriple, targetTriple: targetTriple, - artifactID: self.sdkName ?? recipe.defaultArtifactID(), + artifactID: self.sdkName ?? recipe.defaultArtifactID, isIncremental: self.incremental, isVerbose: self.verbose, logger: logger diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index 7ef9a35..0570c81 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -55,11 +55,11 @@ extension SwiftSDKGenerator { try await self.createDirectoryIfNeeded(at: pathsConfiguration.artifactsCachePath) try await self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath) - let swiftSdkProduct = try await recipe.makeSwiftSDK(generator: self, engine: engine, httpClient: client) + let swiftSDKProduct = try await recipe.makeSwiftSDK(generator: self, engine: engine, httpClient: client) let toolsetJSONPath = try await generateToolsetJSON(recipe: recipe) - try await generateDestinationJSON(toolsetPath: toolsetJSONPath, sdkDirPath: swiftSdkProduct.sdkDirPath) + try await generateDestinationJSON(toolsetPath: toolsetJSONPath, sdkDirPath: swiftSDKProduct.sdkDirPath) try await generateArtifactBundleManifest() diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 58b0bde..7e3db59 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -85,7 +85,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { toolset.librarian = Toolset.ToolProperties(path: "llvm-ar") } - public func defaultArtifactID() -> String { + public var defaultArtifactID: String { """ \(versionsConfiguration.swiftVersion)_\(linuxDistribution.name.rawValue)_\(linuxDistribution.release)_\( mainTargetTriple.cpu.linuxConventionName diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 84892e9..d5b07d2 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -23,8 +23,8 @@ public protocol SwiftSDKRecipe: Sendable { /// Update the given toolset with platform specific options func applyPlatformOptions(toolset: inout Toolset) - /// Returns the default identifier of the Swift SDK - func defaultArtifactID() -> String + /// The default identifier of the Swift SDK + var defaultArtifactID: String { get } /// The main entrypoint of the recipe to make a Swift SDK func makeSwiftSDK(generator: SwiftSDKGenerator, engine: Engine, httpClient: HTTPClient) async throws -> SwiftSDKProduct diff --git a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift index c3db889..def753e 100644 --- a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift +++ b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift @@ -62,7 +62,7 @@ final class ArchitectureMappingTests: XCTestCase { // Linux is currently the only supported runtime environment targetTriple: targetTriple, - artifactID: recipe.defaultArtifactID(), + artifactID: recipe.defaultArtifactID, // Remaining fields are placeholders which are the same for all // combinations of build and runtime architecture isIncremental: false,