From f52aa21ff7726ae49a4e8be1b8904cde051136bd Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Mon, 3 Mar 2025 14:20:25 -0500 Subject: [PATCH 01/15] Increase unzip stream limit for larger package lists --- Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index f16b82a..88d420a 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -21,7 +21,7 @@ public extension ByteBuffer { standardInput: [self].async, collectStandardOutput: true, collectStandardError: false, - perStreamCollectionLimitBytes: 20 * 1024 * 1024 + perStreamCollectionLimitBytes: 100 * 1024 * 1024 ) try result.exitReason.throwIfNonZero() From dedcd6df25cbbb0634671ca34bad3302c7d2603f Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Mon, 3 Mar 2025 21:02:15 -0500 Subject: [PATCH 02/15] Add missing --linux-distribution-version to sdkGeneratorArguments --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 15b19c9..4c2c43f 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -182,7 +182,8 @@ struct SDKConfiguration { "--swift-version \(swiftVersion)-RELEASE", testLinuxSwiftSDKs ? "--host \(hostArch!)-unknown-linux-gnu" : nil, "--target \(architecture)-unknown-linux-gnu", - "--linux-distribution-name \(linuxDistributionName)" + "--linux-distribution-name \(linuxDistributionName)", + "--linux-distribution-version \(linuxDistributionVersion)" ].compactMap{ $0 }.joined(separator: " ") } } From fd589114d186c1a9f9e53eb061ee61d6cab6689f Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sun, 16 Feb 2025 09:48:25 -0500 Subject: [PATCH 03/15] Add Debian 11/12 as potential distributions for building SDKs for - The Debian 11 configuration does not work since there is no official Swift container for it. But, later a special case will be added. --- Sources/GeneratorCLI/GeneratorCLI.swift | 10 ++- .../Generator/SwiftSDKGenerator+Copy.swift | 2 +- .../PlatformModels/LinuxDistribution.swift | 82 +++++++++++++++++-- .../VersionsConfiguration.swift | 4 + 4 files changed, 87 insertions(+), 11 deletions(-) diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index a322f10..f071607 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -191,7 +191,8 @@ extension GeneratorCLI { @Option( help: """ - Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`. + Linux distribution to use if the target platform is Linux. + - Available options: `ubuntu`, `debian`, `rhel`. Default is `ubuntu`. """, transform: LinuxDistribution.Name.init(nameString:) ) @@ -200,8 +201,9 @@ extension GeneratorCLI { @Option( help: """ Version of the Linux distribution used as a target platform. - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). + - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. + - Available options for Debian: `11`, `12` (default when `--linux-distribution-name` is `debian`). + - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). """ ) var linuxDistributionVersion: String? @@ -229,6 +231,8 @@ extension GeneratorCLI { linuxDistributionDefaultVersion = "ubi9" case .ubuntu: linuxDistributionDefaultVersion = "22.04" + case .debian: + linuxDistributionDefaultVersion = "12" } let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion let linuxDistribution = try LinuxDistribution(name: linuxDistributionName, version: linuxDistributionVersion) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 5b9afae..22dba1c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -72,7 +72,7 @@ extension SwiftSDKGenerator { // architecture-specific directories: // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. - if case .ubuntu = targetDistribution { + if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { subpaths += [("\(targetTriple.archName)-linux-gnu", false)] // Custom subpath for armv7 diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index f10b818..4b2190d 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -14,6 +14,7 @@ public enum LinuxDistribution: Hashable, Sendable { public enum Name: String { case rhel case ubuntu + case debian } public enum RHEL: String, Sendable { @@ -34,7 +35,8 @@ public enum LinuxDistribution: Hashable, Sendable { case "24.04": self = .noble default: - throw GeneratorError.unknownLinuxDistribution(name: LinuxDistribution.Name.ubuntu.rawValue, version: version) + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.ubuntu.rawValue, version: version) } } @@ -48,7 +50,8 @@ public enum LinuxDistribution: Hashable, Sendable { public var requiredPackages: [String] { switch self { - case .focal: return [ + case .focal: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -61,7 +64,8 @@ public enum LinuxDistribution: Hashable, Sendable { "zlib1g", "zlib1g-dev", ] - case .jammy: return [ + case .jammy: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -74,7 +78,8 @@ public enum LinuxDistribution: Hashable, Sendable { "zlib1g", "zlib1g-dev", ] - case .noble: return [ + case .noble: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -91,8 +96,64 @@ public enum LinuxDistribution: Hashable, Sendable { } } + public enum Debian: String, Sendable { + case bullseye + case bookworm + + init(version: String) throws { + switch version { + case "11": self = .bullseye + case "12": self = .bookworm + default: + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.debian.rawValue, version: version) + } + } + + var version: String { + switch self { + case .bullseye: return "11" + case .bookworm: return "12" + } + } + + public var requiredPackages: [String] { + switch self { + case .bullseye: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-10-dev", + "libicu66", + "libicu-dev", + "libstdc++-10-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + case .bookworm: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-12-dev", + "libicu72", + "libicu-dev", + "libstdc++-12-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + } + } + } + case rhel(RHEL) case ubuntu(Ubuntu) + case debian(Debian) public init(name: Name, version: String) throws { switch name { @@ -104,6 +165,9 @@ public enum LinuxDistribution: Hashable, Sendable { case .ubuntu: self = try .ubuntu(Ubuntu(version: version)) + + case .debian: + self = try .debian(Debian(version: version)) } } @@ -111,6 +175,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case .rhel: return .rhel case .ubuntu: return .ubuntu + case .debian: return .debian } } @@ -118,6 +183,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return rhel.rawValue case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } @@ -125,12 +191,13 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return "rhel-\(rhel.rawValue)" case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } } -public extension LinuxDistribution.Name { - init(nameString: String) throws { +extension LinuxDistribution.Name { + public init(nameString: String) throws { guard let name = LinuxDistribution.Name(rawValue: nameString) else { throw GeneratorError.unknownLinuxDistribution(name: nameString, version: nil) } @@ -144,7 +211,7 @@ extension LinuxDistribution: CustomStringConvertible { switch self { case .rhel: versionComponent = self.release.uppercased() - case .ubuntu: + case .ubuntu, .debian: versionComponent = self.release.capitalized } @@ -157,6 +224,7 @@ extension LinuxDistribution.Name: CustomStringConvertible { switch self { case .rhel: return "RHEL" case .ubuntu: return "Ubuntu" + case .debian: return "Debian" } } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 85ca59a..8460c24 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -37,6 +37,8 @@ public struct VersionsConfiguration: Sendable { switch self.linuxDistribution { case let .ubuntu(ubuntu): return "ubuntu\(ubuntu.version)\(self.linuxArchSuffix)" + case let .debian(debian): + return "debian\(debian.version)\(self.linuxArchSuffix)" case let .rhel(rhel): return "\(rhel.rawValue)\(self.linuxArchSuffix)" } @@ -55,6 +57,8 @@ public struct VersionsConfiguration: Sendable { switch self.linuxDistribution { case let .ubuntu(ubuntu): computedSubdirectory = "ubuntu\(ubuntu.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" + case let .debian(debian): + computedSubdirectory = "debian\(debian.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" case let .rhel(rhel): computedSubdirectory = rhel.rawValue } From c28cbc026badea63e15e8283d2ecfab21e6fb402 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Tue, 4 Mar 2025 07:58:07 -0500 Subject: [PATCH 04/15] Initial code to download dependencies for Debian distributions from package mirrors --- .../SwiftSDKGenerator+Download.swift | 208 ++++++++++-------- .../PlatformModels/LinuxDistribution.swift | 2 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 31 ++- .../SystemUtils/GeneratorError.swift | 15 +- 4 files changed, 149 insertions(+), 107 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 59db653..a80362a 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -21,8 +21,9 @@ import struct Foundation.URL import struct SystemPackage.FilePath -private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" -private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" +private let ubuntuMainMirror = "http://gb.archive.ubuntu.com/ubuntu" +private let ubuntuPortsMirror = "http://ports.ubuntu.com/ubuntu-ports" +private let debianMirror = "http://deb.debian.org/debian" extension FilePath { var metadataValue: Logger.MetadataValue { @@ -73,34 +74,71 @@ extension SwiftSDKGenerator { ]) } - func downloadUbuntuPackages( + func getMirrorURL(for linuxDistribution: LinuxDistribution) throws -> String { + if linuxDistribution.name == .ubuntu { + if targetTriple.arch == .x86_64 { + return ubuntuMainMirror + } else { + return ubuntuPortsMirror + } + } else if linuxDistribution.name == .debian { + return debianMirror + } else { + throw GeneratorError.distributionSupportsOnlyDockerGenerator(linuxDistribution) + } + } + + func packagesFileName(isXzAvailable: Bool) -> String { + if isXzAvailable { + return "Packages.xz" + } + // Use .gz if xz is not available + return "Packages.gz" + } + + func downloadDebianPackages( _ client: some HTTPClientProtocol, _ engine: QueryEngine, requiredPackages: [String], versionsConfiguration: VersionsConfiguration, sdkDirPath: FilePath ) async throws { - logger.debug("Parsing Ubuntu packages list...") + let mirrorURL = try getMirrorURL(for: versionsConfiguration.linuxDistribution) + let distributionName = versionsConfiguration.linuxDistribution.name + let distributionRelease = versionsConfiguration.linuxDistribution.release // Find xz path let xzPath = try await which("xz") if xzPath == nil { + // If we don't have xz, it's required for Packages.xz for debian + if distributionName == .debian { + throw GeneratorError.debianPackagesListDownloadRequiresXz + } + logger.warning(""" The `xz` utility was not found in `PATH`. \ Consider installing it for more efficient downloading of package lists. """) } - async let mainPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, + logger.info("Downloading and parsing packages lists...", metadata: [ + "distributionName": .stringConvertible(distributionName), "distributionRelease": .string(distributionRelease) + ]) + async let mainPackages = try await parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "", repository: "main", targetTriple: self.targetTriple, isVerbose: self.isVerbose, xzPath: xzPath ) - async let updatesPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, + async let updatesPackages = try await parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, releaseSuffix: "-updates", repository: "main", targetTriple: self.targetTriple, @@ -108,10 +146,12 @@ extension SwiftSDKGenerator { xzPath: xzPath ) - async let universePackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, + async let extraPackages = try await parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, releaseSuffix: "-updates", - repository: "universe", + repository: distributionName == .ubuntu ? "universe" : "contrib", targetTriple: self.targetTriple, isVerbose: self.isVerbose, xzPath: xzPath @@ -119,18 +159,20 @@ extension SwiftSDKGenerator { let allPackages = try await mainPackages .merging(updatesPackages, uniquingKeysWith: { $1 }) - .merging(universePackages, uniquingKeysWith: { $1 }) + .merging(extraPackages, uniquingKeysWith: { $1 }) let urls = requiredPackages.compactMap { allPackages[$0] } guard urls.count == requiredPackages.count else { - throw GeneratorError.ubuntuPackagesParsingFailure( + throw GeneratorError.packagesListParsingFailure( expectedPackages: requiredPackages.count, actual: urls.count ) } - logger.info("Downloading Ubuntu packages...", metadata: ["packageCount": .stringConvertible(urls.count)]) + logger.info("Downloading packages...", metadata: [ + "distributionName": .stringConvertible(distributionName), "packageCount": .stringConvertible(urls.count) + ]) try await inTemporaryDirectory { fs, tmpDir in let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine) await report(downloadedFiles: downloadedFiles) @@ -142,6 +184,68 @@ extension SwiftSDKGenerator { } } + private func parseDebianPackageList( + using client: HTTPClientProtocol, + mirrorURL: String, + release: String, + releaseSuffix: String, + repository: String, + targetTriple: Triple, + isVerbose: Bool, + xzPath: String? + ) async throws -> [String: URL] { + let packagesListURL = """ + \(mirrorURL)/dists/\(release)\(releaseSuffix)/\(repository)/binary-\( + targetTriple.arch!.debianConventionName + )/\(packagesFileName(isXzAvailable: xzPath != nil)) + """ + + logger.debug("Starting download of packages list", metadata: ["packagesListURL": .string(packagesListURL)]) + guard let packages = try await client.downloadDebianPackagesList( + from: packagesListURL, + unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available + isVerbose: isVerbose + ) else { + throw GeneratorError.packagesListDecompressionFailure + } + + let packageRef = Reference(Substring.self) + let pathRef = Reference(Substring.self) + + let regex = Regex { + "Package: " + + Capture(as: packageRef) { + OneOrMore(.anyNonNewline) + } + + OneOrMore(.any, .reluctant) + + "Filename: " + + Capture(as: pathRef) { + OneOrMore(.anyNonNewline) + } + + Anchor.endOfLine + + OneOrMore(.any, .reluctant) + + "MD5sum: " + + OneOrMore(.hexDigit) + } + + var result = [String: URL]() + for match in packages.matches(of: regex) { + guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } + + result[String(match[packageRef])] = url + } + + return result + } + func downloadFiles( from urls: [URL], to directory: FilePath, @@ -185,7 +289,7 @@ extension SwiftSDKGenerator { } extension HTTPClientProtocol { - private func downloadUbuntuPackagesList( + func downloadDebianPackagesList( from url: String, unzipWith zipPath: String, isVerbose: Bool @@ -196,78 +300,4 @@ extension HTTPClientProtocol { return String(buffer: packages) } - - func packagesFileName(isXzAvailable: Bool) -> String { - if isXzAvailable { - return "Packages.xz" - } - // Use .gz if xz is not available - return "Packages.gz" - } - - func parseUbuntuPackagesList( - ubuntuRelease: String, - releaseSuffix: String = "", - repository: String, - targetTriple: Triple, - isVerbose: Bool, - xzPath: String? - ) async throws -> [String: URL] { - let mirrorURL: String - if targetTriple.arch == .x86_64 { - mirrorURL = ubuntuAMD64Mirror - } else { - mirrorURL = ubuntuARM64Mirror - } - - let packagesListURL = """ - \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\( - targetTriple.arch!.debianConventionName - )/\(packagesFileName(isXzAvailable: xzPath != nil)) - """ - - guard let packages = try await downloadUbuntuPackagesList( - from: packagesListURL, - unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available - isVerbose: isVerbose - ) else { - throw GeneratorError.ubuntuPackagesDecompressionFailure - } - - let packageRef = Reference(Substring.self) - let pathRef = Reference(Substring.self) - - let regex = Regex { - "Package: " - - Capture(as: packageRef) { - OneOrMore(.anyNonNewline) - } - - OneOrMore(.any, .reluctant) - - "Filename: " - - Capture(as: pathRef) { - OneOrMore(.anyNonNewline) - } - - Anchor.endOfLine - - OneOrMore(.any, .reluctant) - - "Description-md5: " - - OneOrMore(.hexDigit) - } - - var result = [String: URL]() - for match in packages.matches(of: regex) { - guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } - - result[String(match[packageRef])] = url - } - - return result - } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index 4b2190d..e621b40 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// public enum LinuxDistribution: Hashable, Sendable { - public enum Name: String { + public enum Name: String, Sendable { case rhel case ubuntu case debian diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 66a4dfe..d4981b3 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -248,18 +248,27 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) if !self.shouldUseDocker { - guard case let .ubuntu(version) = linuxDistribution else { - throw GeneratorError - .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) + switch linuxDistribution { + case .ubuntu(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + case .debian(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + default: + throw GeneratorError + .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) } - - try await generator.downloadUbuntuPackages( - client, - engine, - requiredPackages: version.requiredPackages, - versionsConfiguration: self.versionsConfiguration, - sdkDirPath: sdkDirPath - ) } switch self.hostSwiftSource { diff --git a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift index 2ecae65..8634a24 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift @@ -25,8 +25,9 @@ enum GeneratorError: Error { case distributionDoesNotSupportArchitecture(LinuxDistribution, targetArchName: String) case fileDoesNotExist(FilePath) case fileDownloadFailed(URL, String) - case ubuntuPackagesDecompressionFailure - case ubuntuPackagesParsingFailure(expectedPackages: Int, actual: Int) + case debianPackagesListDownloadRequiresXz + case packagesListDecompressionFailure + case packagesListParsingFailure(expectedPackages: Int, actual: Int) } extension GeneratorError: CustomStringConvertible { @@ -59,10 +60,12 @@ extension GeneratorError: CustomStringConvertible { return "Expected to find a file at path `\(filePath)`." case let .fileDownloadFailed(url, status): return "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." - case .ubuntuPackagesDecompressionFailure: - return "Failed to decompress the list of Ubuntu packages" - case let .ubuntuPackagesParsingFailure(expected, actual): - return "Failed to parse Ubuntu packages manifest, expected \(expected), found \(actual) packages." + case .debianPackagesListDownloadRequiresXz: + return "Downloading the Debian packages list requires xz, and it is not installed." + case .packagesListDecompressionFailure: + return "Failed to decompress the list of packages." + case let .packagesListParsingFailure(expected, actual): + return "Failed to parse packages manifest, expected \(expected), found \(actual) packages." } } } From e8aa0c3f1c1e7b285a2f3384ca35794a5845bddd Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Thu, 6 Mar 2025 20:35:48 -0500 Subject: [PATCH 05/15] Support downloading Debian packages on top of Ubuntu packages - When downloading Debian packages, we don't get contrib packages since they are not needed. --- .../SwiftSDKGenerator+Download.swift | 101 +++++++++--------- .../SystemUtils/ByteBuffer+Utils.swift | 6 +- 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index a80362a..9d64c9e 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -124,42 +124,50 @@ extension SwiftSDKGenerator { logger.info("Downloading and parsing packages lists...", metadata: [ "distributionName": .stringConvertible(distributionName), "distributionRelease": .string(distributionRelease) ]) - async let mainPackages = try await parseDebianPackageList( - using: client, - mirrorURL: mirrorURL, - release: distributionRelease, - releaseSuffix: "", - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) - - async let updatesPackages = try await parseDebianPackageList( - using: client, - mirrorURL: mirrorURL, - release: distributionRelease, - releaseSuffix: "-updates", - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) - - async let extraPackages = try await parseDebianPackageList( - using: client, - mirrorURL: mirrorURL, - release: distributionRelease, - releaseSuffix: "-updates", - repository: distributionName == .ubuntu ? "universe" : "contrib", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) - - let allPackages = try await mainPackages - .merging(updatesPackages, uniquingKeysWith: { $1 }) - .merging(extraPackages, uniquingKeysWith: { $1 }) + + let allPackages = try await withThrowingTaskGroup(of: [String: URL].self) { group in + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + if distributionName == .ubuntu { + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "universe", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + } + + var packages: [String : URL] = [String: URL]() + for try await result in group { + packages.merge(result, uniquingKeysWith: { $1 }) + } + return packages + } let urls = requiredPackages.compactMap { allPackages[$0] } @@ -191,20 +199,22 @@ extension SwiftSDKGenerator { releaseSuffix: String, repository: String, targetTriple: Triple, - isVerbose: Bool, xzPath: String? ) async throws -> [String: URL] { + var contextLogger = logger + let packagesListURL = """ \(mirrorURL)/dists/\(release)\(releaseSuffix)/\(repository)/binary-\( targetTriple.arch!.debianConventionName )/\(packagesFileName(isXzAvailable: xzPath != nil)) """ - - logger.debug("Starting download of packages list", metadata: ["packagesListURL": .string(packagesListURL)]) + contextLogger[metadataKey: "packagesListURL"] = .string(packagesListURL) + + contextLogger.debug("Downloading packages list...") guard let packages = try await client.downloadDebianPackagesList( from: packagesListURL, unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available - isVerbose: isVerbose + logger: logger ) else { throw GeneratorError.packagesListDecompressionFailure } @@ -228,14 +238,9 @@ extension SwiftSDKGenerator { } Anchor.endOfLine - - OneOrMore(.any, .reluctant) - - "MD5sum: " - - OneOrMore(.hexDigit) } + contextLogger.debug("Processing packages list...") var result = [String: URL]() for match in packages.matches(of: regex) { guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } @@ -292,9 +297,9 @@ extension HTTPClientProtocol { func downloadDebianPackagesList( from url: String, unzipWith zipPath: String, - isVerbose: Bool + logger: Logger ) async throws -> String? { - guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) else { + guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, logger: logger) else { throw FileOperationError.downloadFailed(url) } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index 88d420a..78b8a56 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -13,15 +13,17 @@ import AsyncProcess import Foundation import NIOCore +import Logging public extension ByteBuffer { - func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { + func unzip(zipPath: String, logger: Logger) async throws -> ByteBuffer? { let result = try await ProcessExecutor.runCollectingOutput( executable: zipPath, ["-cd"], standardInput: [self].async, collectStandardOutput: true, collectStandardError: false, - perStreamCollectionLimitBytes: 100 * 1024 * 1024 + perStreamCollectionLimitBytes: 100 * 1024 * 1024, + logger: logger ) try result.exitReason.throwIfNonZero() From da140b3dc2715b9b9c224b57549555cfe1d15736 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Thu, 6 Mar 2025 21:30:36 -0500 Subject: [PATCH 06/15] Add support for Debian 11 build from packages --- .../PlatformModels/VersionsConfiguration.swift | 11 +++++++++-- .../SwiftSDKRecipes/LinuxRecipe.swift | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 8460c24..af6cda0 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import struct Foundation.URL +import Logging public struct VersionsConfiguration: Sendable { init( @@ -18,12 +19,18 @@ public struct VersionsConfiguration: Sendable { swiftBranch: String? = nil, lldVersion: String, linuxDistribution: LinuxDistribution, - targetTriple: Triple + targetTriple: Triple, + logger: Logger ) throws { self.swiftVersion = swiftVersion self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" self.lldVersion = lldVersion - self.linuxDistribution = linuxDistribution + if case let .debian(debian) = linuxDistribution, debian.version == "11" { + logger.warning("Debian 11 selected but not officially supported by Swift, falling back on Ubuntu 20.04 toolchain...") + self.linuxDistribution = try .init(name: .ubuntu, version: "20.04") + } else { + self.linuxDistribution = linuxDistribution + } self.linuxArchSuffix = targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index d4981b3..95ecea6 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -62,7 +62,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { swiftBranch: swiftBranch, lldVersion: lldVersion, linuxDistribution: linuxDistribution, - targetTriple: targetTriple + targetTriple: targetTriple, + logger: logger ) let targetSwiftSource: LinuxRecipe.TargetSwiftSource From a41ce54b088784e7bbaafee3c68163f280892c88 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Fri, 7 Mar 2025 14:33:22 -0500 Subject: [PATCH 07/15] Add EndToEndTests for all supported Debian Swift SDK builds --- .../EndToEndTests.swift | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 4c2c43f..a40702c 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -163,6 +163,12 @@ struct SDKConfiguration { return res } + func withLinuxDistributionVersion(_ version: String) -> SDKConfiguration { + var res = self + res.linuxDistributionVersion = version + return res + } + func withArchitecture(_ arch: String) -> SDKConfiguration { var res = self res.architecture = arch @@ -396,6 +402,112 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { } } +final class Swift59_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "debian", + linuxDistributionVersion: "11", // we use ubuntu2004 toolchain here + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + // NOTE: we do support building a Swift SDK from a pre-built Debian 11 Bullseye container +} + +final class Swift510_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64")) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64")) + } + + // NOTE: we do support building a Swift SDK from a pre-built Debian 11 Bullseye container +} + +final class Swift60_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.3", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64")) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64")) + } + + // NOTE: we do support building a Swift SDK from a pre-built Debian 11 Bullseye container +} + final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", From 051985ae04c3393c52deaec67d8e87cc08bf636d Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sat, 22 Mar 2025 21:00:53 -0400 Subject: [PATCH 08/15] Restore missing withContainerImageSuffix --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index fdde147..8afca92 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -179,6 +179,12 @@ struct SDKConfiguration { return res } + func withContainerImageSuffix(_ containerImageSuffix: String) -> SDKConfiguration { + var res = self + res.containerImageSuffix = containerImageSuffix + return res + } + func withLinuxDistributionVersion(_ version: String) -> SDKConfiguration { var res = self res.linuxDistributionVersion = version From ea57421a55fb5a1aef77cca40b8e2ac3c2c2d84a Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sun, 23 Mar 2025 07:52:34 -0400 Subject: [PATCH 09/15] Update comments for Debian 11 end-to-end tests --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 8afca92..d8177d5 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -516,7 +516,8 @@ final class Swift510_DebianEndToEndTests: XCTestCase { config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64")) } - // NOTE: we do support building a Swift SDK from a pre-built Debian 11 Bullseye container + // NOTE: the generator does not support building a Debian 11 Swift container automatically, + // so we don't test this scenario. } final class Swift60_DebianEndToEndTests: XCTestCase { @@ -560,7 +561,8 @@ final class Swift60_DebianEndToEndTests: XCTestCase { config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64")) } - // NOTE: we do support building a Swift SDK from a pre-built Debian 11 Bullseye container + // NOTE: the generator does not support building a Debian 11 Swift container automatically, + // so we don't test this scenario.} } final class Swift59_RHELEndToEndTests: XCTestCase { From 72588acd6e7a7458d173eaae2b73b3c4d23afaef Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sun, 23 Mar 2025 13:58:20 -0400 Subject: [PATCH 10/15] Support building Swift 5.9 SDKs for Debian Bookworm using Ubuntu Jammy toolchain --- .../PlatformModels/LinuxDistribution.swift | 2 +- .../VersionsConfiguration.swift | 46 +++++++++---------- .../EndToEndTests.swift | 19 ++++++-- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index e621b40..e8d26cb 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -125,7 +125,7 @@ public enum LinuxDistribution: Hashable, Sendable { "libc6-dev", "libgcc-s1", "libgcc-10-dev", - "libicu66", + "libicu67", "libicu-dev", "libstdc++-10-dev", "libstdc++6", diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 61d5d57..08292c5 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -26,16 +26,10 @@ public struct VersionsConfiguration: Sendable { self.swiftVersion = swiftVersion self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" self.lldVersion = lldVersion - if case let .debian(debian) = linuxDistribution, debian.version == "11" { - logger.warning( - "Debian 11 selected but not officially supported by Swift, falling back on Ubuntu 20.04 toolchain..." - ) - self.linuxDistribution = try .init(name: .ubuntu, version: "20.04") - } else { - self.linuxDistribution = linuxDistribution - } + self.linuxDistribution = linuxDistribution self.linuxArchSuffix = targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + self.logger = logger } let swiftVersion: String @@ -43,12 +37,27 @@ public struct VersionsConfiguration: Sendable { let lldVersion: String let linuxDistribution: LinuxDistribution let linuxArchSuffix: String + private let logger: Logger var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): return "ubuntu\(ubuntu.version)\(self.linuxArchSuffix)" case let .debian(debian): + if debian.version == "11" { + logger.warning( + "Debian 11 selected but not officially supported by Swift, falling back on Ubuntu 20.04 toolchain..." + ) + // Ubuntu 20.04 toolchain is binary compatible with Debian 11 + return "ubuntu20.04\(self.linuxArchSuffix)" + } else if self.swiftVersion.hasPrefix("5.9") || self.swiftVersion == "5.10" { + logger.warning( + "Debian 12 selected but not officially supported by Swift version, falling back on Ubuntu 22.04 toolchain...", + metadata: ["swiftVersion": .string(self.swiftVersion)] + ) + // Ubuntu 22.04 toolchain is binary compatible with Debian 12 + return "ubuntu22.04\(self.linuxArchSuffix)" + } return "debian\(debian.version)\(self.linuxArchSuffix)" case let .rhel(rhel): return "\(rhel.rawValue)\(self.linuxArchSuffix)" @@ -56,7 +65,7 @@ public struct VersionsConfiguration: Sendable { } func swiftDistributionName(platform: String? = nil) -> String { - "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatform)" + return "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatform)" } func swiftDownloadURL( @@ -64,25 +73,16 @@ public struct VersionsConfiguration: Sendable { platform: String? = nil, fileExtension: String = "tar.gz" ) -> URL { - let computedSubdirectory: String - switch self.linuxDistribution { - case let .ubuntu(ubuntu): - computedSubdirectory = - "ubuntu\(ubuntu.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" - case let .debian(debian): - computedSubdirectory = - "debian\(debian.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" - case let .rhel(rhel): - computedSubdirectory = rhel.rawValue - } + let computedPlatform = platform ?? self.swiftPlatform + let computedSubdirectory = + subdirectory ?? computedPlatform.replacingOccurrences(of: ".", with: "") return URL( string: """ https://download.swift.org/\( self.swiftBranch - )/\( - subdirectory ?? computedSubdirectory - )/swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: platform)).\(fileExtension) + )/\(computedSubdirectory)/\ + swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: computedPlatform)).\(fileExtension) """ )! } diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index d8177d5..39be681 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -462,17 +462,30 @@ final class Swift59_DebianEndToEndTests: XCTestCase { withDocker: false ) - func testAarch64Direct() async throws { + func testBullseyeAarch64Direct() async throws { try skipSlow() try await buildTestcases(config: config.withArchitecture("aarch64")) } - func testX86_64Direct() async throws { + func testBullseyeX86_64Direct() async throws { try skipSlow() try await buildTestcases(config: config.withArchitecture("x86_64")) } - // NOTE: we do support building a Swift SDK from a pre-built Debian 11 Bullseye container + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("x86_64")) + } + + // NOTE: the generator does not support building a Debian 11/Debian 12 Swift SDK for + // Swift 5.9.x and 5.10 without a pre-built container, so we do not test this here. } final class Swift510_DebianEndToEndTests: XCTestCase { From f4d092297b28d19d18554a82e0caca03a68366e5 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sun, 23 Mar 2025 15:00:34 -0400 Subject: [PATCH 11/15] Adjust targetSwift path to include swiftPlatform to identify the toolchain - Otherwise weird things happen from the EndToEndTests when running tests that then need to re-download a new toolchain on top. --- .../Artifacts/DownloadableArtifacts.swift | 10 ++++--- .../VersionsConfiguration.swift | 29 +++++++++---------- .../EndToEndTests.swift | 12 ++++---- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index ec4725e..1af7fd1 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -54,12 +54,12 @@ struct DownloadableArtifacts: Sendable { if hostTriple.os == .linux { // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts - let linuxArchSuffix = + let hostArchSuffix = hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" self.hostSwift = .init( remoteURL: versions.swiftDownloadURL( - subdirectory: "amazonlinux2\(linuxArchSuffix)", - platform: "amazonlinux2\(linuxArchSuffix)", + subdirectory: "amazonlinux2\(hostArchSuffix)", + platform: "amazonlinux2\(hostArchSuffix)", fileExtension: "tar.gz" ), localPath: paths.artifactsCachePath @@ -97,7 +97,9 @@ struct DownloadableArtifacts: Sendable { self.targetSwift = .init( remoteURL: versions.swiftDownloadURL(), localPath: paths.artifactsCachePath - .appending("target_swift_\(versions.swiftVersion)_\(targetTriple.triple).tar.gz"), + .appending( + "target_swift_\(versions.swiftVersion)_\(versions.swiftPlatform)_\(targetTriple.archName).tar.gz" + ), isPrebuilt: true ) } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 08292c5..3459126 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -42,30 +42,28 @@ public struct VersionsConfiguration: Sendable { var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): - return "ubuntu\(ubuntu.version)\(self.linuxArchSuffix)" + return "ubuntu\(ubuntu.version)" case let .debian(debian): if debian.version == "11" { - logger.warning( - "Debian 11 selected but not officially supported by Swift, falling back on Ubuntu 20.04 toolchain..." - ) // Ubuntu 20.04 toolchain is binary compatible with Debian 11 - return "ubuntu20.04\(self.linuxArchSuffix)" + return "ubuntu20.04" } else if self.swiftVersion.hasPrefix("5.9") || self.swiftVersion == "5.10" { - logger.warning( - "Debian 12 selected but not officially supported by Swift version, falling back on Ubuntu 22.04 toolchain...", - metadata: ["swiftVersion": .string(self.swiftVersion)] - ) // Ubuntu 22.04 toolchain is binary compatible with Debian 12 - return "ubuntu22.04\(self.linuxArchSuffix)" + return "ubuntu22.04" } - return "debian\(debian.version)\(self.linuxArchSuffix)" + return "debian\(debian.version)" case let .rhel(rhel): - return "\(rhel.rawValue)\(self.linuxArchSuffix)" + return rhel.rawValue } } + var swiftPlatformAndSuffix: String { + return "\(self.swiftPlatform)\(self.linuxArchSuffix)" + } + func swiftDistributionName(platform: String? = nil) -> String { - return "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatform)" + return + "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatformAndSuffix)" } func swiftDownloadURL( @@ -73,9 +71,10 @@ public struct VersionsConfiguration: Sendable { platform: String? = nil, fileExtension: String = "tar.gz" ) -> URL { - let computedPlatform = platform ?? self.swiftPlatform + let computedPlatform = platform ?? self.swiftPlatformAndSuffix let computedSubdirectory = - subdirectory ?? computedPlatform.replacingOccurrences(of: ".", with: "") + subdirectory + ?? computedPlatform.replacingOccurrences(of: ".", with: "") return URL( string: """ diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 39be681..e88aedf 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -484,8 +484,8 @@ final class Swift59_DebianEndToEndTests: XCTestCase { config: config.withLinuxDistributionVersion("12").withArchitecture("x86_64")) } - // NOTE: the generator does not support building a Debian 11/Debian 12 Swift SDK for - // Swift 5.9.x and 5.10 without a pre-built container, so we do not test this here. + // NOTE: the generator does not support building a Debian 11/Debian 12 Swift SDK with Docker + // for Swift 5.9.x and 5.10 without a pre-built container, so we do not test this here. } final class Swift510_DebianEndToEndTests: XCTestCase { @@ -529,8 +529,8 @@ final class Swift510_DebianEndToEndTests: XCTestCase { config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64")) } - // NOTE: the generator does not support building a Debian 11 Swift container automatically, - // so we don't test this scenario. + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. } final class Swift60_DebianEndToEndTests: XCTestCase { @@ -574,8 +574,8 @@ final class Swift60_DebianEndToEndTests: XCTestCase { config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64")) } - // NOTE: the generator does not support building a Debian 11 Swift container automatically, - // so we don't test this scenario.} + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. } final class Swift59_RHELEndToEndTests: XCTestCase { From 5462ab077b534ce984ef3aa2aecbe364ed5a4666 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Mon, 24 Mar 2025 13:15:59 -0400 Subject: [PATCH 12/15] Add tests for downloading Ubuntu toolchains when Debian targets are selected --- .../SwiftSDKRecipes/LinuxRecipeTests.swift | 120 ++++++++++++++++-- 1 file changed, 111 insertions(+), 9 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 656600e..c4851dc 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -20,6 +20,7 @@ final class LinuxRecipeTests: XCTestCase { func createRecipe( hostTriple: Triple = Triple("x86_64-unknown-linux-gnu"), + linuxDistribution: LinuxDistribution, swiftVersion: String = "6.0", withDocker: Bool = false, fromContainerImage: String? = nil, @@ -30,7 +31,7 @@ final class LinuxRecipeTests: XCTestCase { try LinuxRecipe( targetTriple: Triple("aarch64-unknown-linux-gnu"), hostTriple: hostTriple, - linuxDistribution: .init(name: .ubuntu, version: "22.04"), + linuxDistribution: linuxDistribution, swiftVersion: swiftVersion, swiftBranch: nil, lldVersion: "", withDocker: withDocker, fromContainerImage: fromContainerImage, @@ -73,8 +74,10 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { - let recipe = try self.createRecipe(swiftVersion: testCase.swiftVersion) + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, swiftVersion: testCase.swiftVersion) var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, targetTriple: testCase.targetTriple @@ -87,7 +90,9 @@ final class LinuxRecipeTests: XCTestCase { } func testToolOptionsForPreinstalledSdk() throws { - let recipe = try self.createRecipe(includeHostToolchain: false) + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, includeHostToolchain: false) var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( toolset: &toolset, targetTriple: Triple("x86_64-unknown-linux-gnu") @@ -137,6 +142,7 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForMacOSHost() throws { let hostTriple = Triple("x86_64-apple-macos") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases: [( recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, @@ -144,14 +150,16 @@ final class LinuxRecipeTests: XCTestCase { )] = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, linuxDistribution: linuxDistribution, swiftVersion: "5.10"), includesHostLLVM: true, includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, linuxDistribution: linuxDistribution, swiftVersion: "6.0"), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: true @@ -159,7 +167,8 @@ final class LinuxRecipeTests: XCTestCase { ( // Remote target tarball with preinstalled toolchain recipe: try createRecipe( - hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), + hostTriple: hostTriple, linuxDistribution: linuxDistribution, swiftVersion: "5.9", + includeHostToolchain: false), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: false @@ -168,6 +177,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -180,6 +190,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -202,23 +213,27 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForLinuxHost() throws { let hostTriple = Triple("x86_64-unknown-linux-gnu") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, linuxDistribution: linuxDistribution, swiftVersion: "5.10"), includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, linuxDistribution: linuxDistribution, swiftVersion: "6.0"), includesTargetSwift: true, includesHostSwift: true ), ( // Remote target tarball with preinstalled toolchain recipe: try createRecipe( - hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), + hostTriple: hostTriple, linuxDistribution: linuxDistribution, swiftVersion: "5.9", + includeHostToolchain: false), includesTargetSwift: true, includesHostSwift: false ), @@ -226,6 +241,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -237,6 +253,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -256,6 +273,89 @@ final class LinuxRecipeTests: XCTestCase { } } + // Ubuntu toolchains will be selected for Debian 11 and 12 depending on the Swift version + func testItemsToDownloadForDebianTargets() throws { + let hostTriple = Triple("x86_64-unknown-linux-gnu") + let testCases = [ + ( + // Debian 11 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.9 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 12 with Swift 5.10 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 11 with Swift 6.0 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.10.1 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10.1" + ), + expectedTargetSwift: "debian12" + ), + ( + // Debian 12 with Swift 6.0 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "debian12" + ), + ] + + for testCase in testCases { + + let pathsConfiguration = PathsConfiguration( + sourceRoot: ".", + artifactID: "my-sdk", + targetTriple: testCase.recipe.mainTargetTriple + ) + let downloadableArtifacts = try DownloadableArtifacts( + hostTriple: testCase.recipe.mainHostTriple, + targetTriple: testCase.recipe.mainTargetTriple, + testCase.recipe.versionsConfiguration, + pathsConfiguration + ) + let itemsToDownload = testCase.recipe.itemsToDownload(from: downloadableArtifacts) + let targetSwiftRemoteURL = itemsToDownload.first(where: { + $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL + })?.remoteURL.absoluteString + + // If this is a Linux host, we do not download LLVM + XCTAssert(targetSwiftRemoteURL!.contains(testCase.expectedTargetSwift)) + } + } + func testHostTriples() throws { let allHostTriples = [ Triple("x86_64-unknown-linux-gnu"), @@ -273,8 +373,10 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { let recipe = try createRecipe( + linuxDistribution: linuxDistribution, swiftVersion: testCase.swiftVersion, includeHostToolchain: testCase.includeHostToolchain ) From 9bf533ba3130d64cda7cd686353e59b1d075b491 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Tue, 25 Mar 2025 20:38:14 -0400 Subject: [PATCH 13/15] Also update README.md to recognize new Debian 11/12 support --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5ec9617..4894c5c 100644 --- a/README.md +++ b/README.md @@ -47,14 +47,14 @@ The generator also allows cross-compiling between any Linux distributions offici | macOS (arm64) | ✅ macOS 13.0+ | ❌ | | macOS (x86_64) | ✅ macOS 13.0+[^1] | ❌ | | Ubuntu | ✅ 20.04+ | ✅ 20.04+ | -| RHEL | ✅ Fedora 39[^2], UBI 9 | ✅ UBI 9 | +| Debian | ✅ 11, 12[^2] | ✅ 11, 12[^2] | +| RHEL | ✅ Fedora 39, UBI 9 | ✅ Fedora 39, UBI 9[^3] | | Amazon Linux 2 | ✅ Supported | ✅ Supported[^3] | -| Debian 12 | ✅ Supported[^2] | ✅ Supported[^2][^3] | [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). -[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms. +[^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible. [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138). ## How to use it From 144b4074731898dc8981ecf85e56172e04e232af Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 26 Mar 2025 09:45:27 -0400 Subject: [PATCH 14/15] Copy /lib and /lib64 for Debian 11 containers - This is required since Debian 11 has a different layout than other Ubuntu/Debian versions. --- .../Generator/SwiftSDKGenerator+Copy.swift | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index ca6724e..c9ad326 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -22,6 +22,7 @@ extension SwiftSDKGenerator { logger.info("Launching a container to extract the Swift SDK for the target triple...") try await withDockerContainer(fromImage: baseDockerImage) { containerID in try await inTemporaryDirectory { generator, _ in + let sdkLibPath = sdkDirPath.appending("lib") let sdkUsrPath = sdkDirPath.appending("usr") try await generator.createDirectoryIfNeeded(at: sdkUsrPath) try await generator.copyFromDockerContainer( @@ -60,6 +61,15 @@ extension SwiftSDKGenerator { to: sdkUsrLib64Path ) try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") + } else if case let containerLib64 = FilePath("/lib64"), + try await generator.doesPathExist(containerLib64, inContainer: containerID) + { + let sdkLib64Path = sdkDirPath.appending("lib64") + try await generator.copyFromDockerContainer( + id: containerID, + from: containerLib64, + to: sdkLib64Path + ) } let sdkUsrLibPath = sdkUsrPath.appending("lib") @@ -73,12 +83,24 @@ extension SwiftSDKGenerator { // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { - subpaths += [("\(targetTriple.archName)-linux-gnu", false)] + let archSubpath = + switch targetTriple.archName { + case "armv7": "arm-linux-gnueabihf" + default: "\(targetTriple.archName)-linux-gnu" + } - // Custom subpath for armv7 - if targetTriple.archName == "armv7" { - subpaths += [("arm-linux-gnueabihf", false)] + // Copy /lib/ for Debian 11 + if case let .debian(debian) = targetDistribution, debian == .bullseye { + try await generator.createDirectoryIfNeeded(at: sdkLibPath) + try await generator.copyFromDockerContainer( + id: containerID, + from: FilePath("/lib").appending(archSubpath), + to: sdkLibPath.appending(archSubpath), + failIfNotExists: false + ) } + + subpaths += [(archSubpath, false)] } for (subpath, failIfNotExists) in subpaths { @@ -89,7 +111,11 @@ extension SwiftSDKGenerator { failIfNotExists: failIfNotExists ) } - try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib") + + // Symlink if we do not have a /lib directory in the SDK + if await !generator.doesFileExist(at: sdkLibPath) { + try await generator.createSymlink(at: sdkLibPath, pointingTo: "usr/lib") + } // Look for 32-bit libraries to remove from RHEL-based distros // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up From b199eefbb077043367d8ab3c1a774acb083f9a20 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Fri, 28 Mar 2025 17:54:08 -0400 Subject: [PATCH 15/15] Fix missing syntax for switch statement in 5.8 --- .../Generator/SwiftSDKGenerator+Copy.swift | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index c9ad326..3b98fc8 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -83,11 +83,12 @@ extension SwiftSDKGenerator { // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { - let archSubpath = - switch targetTriple.archName { - case "armv7": "arm-linux-gnueabihf" - default: "\(targetTriple.archName)-linux-gnu" - } + var archSubpath = "\(targetTriple.archName)-linux-gnu" + + // armv7 with Debian uses a custom subpath for armhf + if targetTriple.archName == "armv7" { + archSubpath = "arm-linux-gnueabihf" + } // Copy /lib/ for Debian 11 if case let .debian(debian) = targetDistribution, debian == .bullseye {