diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..fff448d --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,18 @@ +name: Pull request + +on: + pull_request: + types: [opened, reopened, synchronize] + +jobs: + tests: + name: Test + uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main + with: + linux_pre_build_command: apt-get update && apt-get install -y locales locales-all libsqlite3-dev + enable_windows_checks: false + soundness: + name: Soundness + uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main + with: + api_breakage_check_enabled: false diff --git a/.license_header_template b/.license_header_template new file mode 100644 index 0000000..3f564ac --- /dev/null +++ b/.license_header_template @@ -0,0 +1,12 @@ +@@===----------------------------------------------------------------------===@@ +@@ +@@ This source file is part of the Swift open source project +@@ +@@ Copyright (c) YEARS Apple Inc. and the Swift project authors +@@ Licensed under Apache License v2.0 with Runtime Library Exception +@@ +@@ See https://swift.org/LICENSE.txt for license information +@@ See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +@@ +@@===----------------------------------------------------------------------===@@ + diff --git a/.licenseignore b/.licenseignore new file mode 100644 index 0000000..1957e72 --- /dev/null +++ b/.licenseignore @@ -0,0 +1,22 @@ +.gitignore +.swiftformat +.swiftformatignore +.editorconfig +.unacceptablelanguageignore +Brewfile +Package.swift +Package.resolved +*.md +*.txt +*.yml +**/.editorconfig +**/*.docc/** +**/*.entitlements +**/*.input +**/*.modulemap +**/*.plist +**/*.xcodeproj/** +**/CODEOWNERS +**/Dockerfile +**/Package.swift +Utilities/git.commit.template diff --git a/.swift-format b/.swift-format new file mode 100644 index 0000000..e9b435d --- /dev/null +++ b/.swift-format @@ -0,0 +1,70 @@ +{ + "fileScopedDeclarationPrivacy" : { + "accessLevel" : "private" + }, + "indentConditionalCompilationBlocks" : true, + "indentSwitchCaseLabels" : false, + "indentation" : { + "spaces" : 2 + }, + "lineBreakAroundMultilineExpressionChainComponents" : false, + "lineBreakBeforeControlFlowKeywords" : false, + "lineBreakBeforeEachArgument" : true, + "lineBreakBeforeEachGenericRequirement" : false, + "lineLength" : 120, + "maximumBlankLines" : 1, + "multiElementCollectionTrailingCommas" : true, + "noAssignmentInExpressions" : { + "allowedFunctions" : [ + "XCTAssertNoThrow" + ] + }, + "prioritizeKeepingFunctionOutputTogether" : false, + "respectsExistingLineBreaks" : true, + "rules" : { + "AllPublicDeclarationsHaveDocumentation" : false, + "AlwaysUseLiteralForEmptyCollectionInit" : false, + "AlwaysUseLowerCamelCase" : false, + "AmbiguousTrailingClosureOverload" : true, + "BeginDocumentationCommentWithOneLineSummary" : false, + "DoNotUseSemicolons" : true, + "DontRepeatTypeInStaticProperties" : true, + "FileScopedDeclarationPrivacy" : true, + "FullyIndirectEnum" : true, + "GroupNumericLiterals" : true, + "IdentifiersMustBeASCII" : true, + "NeverForceUnwrap" : false, + "NeverUseForceTry" : false, + "NeverUseImplicitlyUnwrappedOptionals" : false, + "NoAccessLevelOnExtensionDeclaration" : true, + "NoAssignmentInExpressions" : true, + "NoBlockComments" : true, + "NoCasesWithOnlyFallthrough" : true, + "NoEmptyTrailingClosureParentheses" : true, + "NoLabelsInCasePatterns" : true, + "NoLeadingUnderscores" : false, + "NoParensAroundConditions" : true, + "NoPlaygroundLiterals" : true, + "NoVoidReturnOnFunctionSignature" : true, + "OmitExplicitReturns" : false, + "OneCasePerLine" : true, + "OneVariableDeclarationPerLine" : true, + "OnlyOneTrailingClosureArgument" : true, + "OrderedImports" : true, + "ReplaceForEachWithForLoop" : true, + "ReturnVoidInsteadOfEmptyTuple" : true, + "TypeNamesShouldBeCapitalized" : true, + "UseEarlyExits" : false, + "UseExplicitNilCheckInConditions" : true, + "UseLetInEveryBoundCaseVariable" : false, + "UseShorthandTypeNames" : true, + "UseSingleLinePropertyGetter" : true, + "UseSynthesizedInitializer" : true, + "UseTripleSlashForDocumentationComments" : true, + "UseWhereClausesInForLoops" : false, + "ValidateDocumentationComments" : false + }, + "spacesAroundRangeFormationOperators" : false, + "tabWidth" : 8, + "version" : 1 +} diff --git a/.swiftformatignore b/.swiftformatignore new file mode 100644 index 0000000..ebdee05 --- /dev/null +++ b/.swiftformatignore @@ -0,0 +1,2 @@ +Sources/Helpers/Vendor/* +Sources/AsyncProcess/ProcessExecutor.swift diff --git a/.unacceptablelanguageignore b/.unacceptablelanguageignore new file mode 100644 index 0000000..24ab69f --- /dev/null +++ b/.unacceptablelanguageignore @@ -0,0 +1,2 @@ +Sources/AsyncProcess/ProcessExecutor.swift +Tests/AsyncProcessTests/IntegrationTests.swift diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2acd53b..10ce9c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,29 +58,6 @@ We require that your commit messages match our template. The easiest way to do t git config commit.template Utilities/git.commit.template -### Run `./Utilities/soundness.sh` - -The scripts directory contains a [`soundness.sh` script](https://github.com/apple/swift-sdk-generator/blob/main/Utilities/soundness.sh) -that enforces additional checks, like license headers and formatting style. - -Please make sure to run `./Utilities/soundness.sh` before pushing a change upstream, otherwise it is likely the PR validation will fail -on minor changes such as formatting issues. - -For frequent contributors, we recommend adding the script as a [git pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks), which you can do via executing the following command in the project root directory: - -```bash -cat << EOF > .git/hooks/pre-push - -if [[ -f "Utilities/soundness.sh" ]]; then - Utilities/soundness.sh -fi -EOF -``` - -Which makes the script execute, and only allow the `git push` to complete if the check has passed. - -In the case of formatting issues, you can then `git add` the formatting changes, and attempt the push again. - ## How to contribute your work -Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. \ No newline at end of file +Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. diff --git a/Docker/docker-compose.yaml b/Docker/docker-compose.yaml index 3db9408..95378ce 100644 --- a/Docker/docker-compose.yaml +++ b/Docker/docker-compose.yaml @@ -19,10 +19,6 @@ services: - ..:/code:z working_dir: /code - soundness: - <<: *common - command: /bin/bash -xcl "swift -version && uname -a && ./Utilities/soundness.sh" - test: <<: *common environment: diff --git a/Package.swift b/Package.swift index 8094ef8..479975e 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.8 +// swift-tools-version: 5.9 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription @@ -11,7 +11,7 @@ let package = Package( .executable( name: "swift-sdk-generator", targets: ["GeneratorCLI"] - ), + ) ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. @@ -23,7 +23,7 @@ let package = Package( .product(name: "ArgumentParser", package: "swift-argument-parser"), ], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), + .enableExperimentalFeature("StrictConcurrency=complete") ] ), .target( @@ -38,22 +38,22 @@ let package = Package( ], exclude: ["Dockerfiles"], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), + .enableExperimentalFeature("StrictConcurrency=complete") ] ), .testTarget( name: "SwiftSDKGeneratorTests", dependencies: [ - "SwiftSDKGenerator", + "SwiftSDKGenerator" ], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), + .enableExperimentalFeature("StrictConcurrency=complete") ] ), .testTarget( name: "GeneratorEngineTests", dependencies: [ - "Helpers", + "Helpers" ] ), .target( @@ -66,13 +66,13 @@ let package = Package( ], exclude: ["Vendor/README.md"], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), + .enableExperimentalFeature("StrictConcurrency=complete") ] ), .testTarget( name: "HelpersTests", dependencies: [ - "Helpers", + "Helpers" ] ), .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"), diff --git a/README.md b/README.md index d456d4d..0feed34 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,23 @@ a newly generated Swift SDK for users to install. ## Requirements -Usage of Swift SDKs requires Swift 5.9, follow [installation instructions on swift.org](https://www.swift.org/install/) to install it first. +Usage of Swift SDKs requires Swift 6.0 or later (experimental implementation is available in Swift 5.9/5.10), follow [installation instructions on swift.org](https://www.swift.org/install/) to install it first. + +After that, verify that the `sdk` command is available: + +``` +swift sdk list +``` + +For Swift 5.9/5.10 add `experimental-` prefix to the command invocation: -After that, verify that the `experimental-sdk` command is available: ``` swift experimental-sdk list ``` The output will either state that no Swift SDKs are available, or produce a list of those you previously had -installed, in case you've used the `swift experimental-sdk install` command before. +installed, in case you've used the `swift sdk install` (or `swift experimental-sdk install` with Swift 5.9/5.10) command before. ### macOS Requirements @@ -47,14 +54,14 @@ The generator also allows cross-compiling between any Linux distributions offici | macOS (arm64) | ✅ macOS 13.0+ | ❌ | | macOS (x86_64) | ✅ macOS 13.0+[^1] | ❌ | | Ubuntu | ✅ 20.04+ | ✅ 20.04+ | -| RHEL | ✅ Fedora 39[^2], UBI 9 | ✅ UBI 9 | +| Debian | ✅ 11, 12[^2] | ✅ 11, 12[^2] | +| RHEL | ✅ Fedora 39, UBI 9 | ✅ Fedora 39, UBI 9[^3] | | Amazon Linux 2 | ✅ Supported | ✅ Supported[^3] | -| Debian 12 | ✅ Supported[^2] | ✅ Supported[^2][^3] | [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). -[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms. +[^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible. [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138). ## How to use it @@ -78,7 +85,7 @@ swift run swift-sdk-generator make-linux-sdk --help After installing a Swift SDK, verify that it's available to SwiftPM: ``` -swift experimental-sdk list +swift sdk list ``` The output of the last command should contain `ubuntu22.04`. Note the full Swift SDK ID in the output, we'll refer to it @@ -87,12 +94,18 @@ subsequently as ``. Create a new project to verify that the SDK works: ``` -mkdir cross-compilation test +mkdir cross-compilation-test cd cross-compilation-test swift package init --type executable ``` -Build this project with the SDK: +Build this project with the Swift SDK: + +``` +swift build --swift-sdk +``` + +When cross-compiling with Swift 5.9/5.10, add `experimental-` prefix to the option name: ``` swift build --experimental-swift-sdk @@ -112,9 +125,9 @@ dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, ``` You can then copy this binary to a Docker image that has Swift runtime libraries installed. For example, -for Ubuntu Jammy and Swift 5.9 this would be `swift:5.9-jammy-slim`. If you'd like to copy the binary to +for Ubuntu Jammy and Swift 6.0 this would be `swift:6.0-jammy-slim`. If you'd like to copy the binary to an arbitrary Ubuntu Jammy system, make sure you pass `--static-swift-stdlib` flag to `swift build`, in addition -to the `--experimental-swift-sdk` option. +to the `--swift-sdk` option. ## Building an SDK from a container image @@ -169,9 +182,22 @@ swift run swift-sdk-generator make-linux-sdk --with-docker --from-container-imag ## Swift SDK distribution The `.artifactbundle` directory produced in the previous section can be packaged as a `.tar.gz` archive and redistributed -in this form. Users of such Swift SDK bundle archive can easily install it with `swift experimental-sdk install` +in this form. Users of such Swift SDK bundle archive can easily install it with `swift sdk install` command, which supports both local file system paths and public `http://` and `https://` URLs as an argument. +To make an `.artifactbundle.tar.gz` archive installable directly from `http://` and `https://` URLs, a checksum must be +generated. This can be done using the `swift package compute-checksum` command, like this: + +```console +swift package compute-checksum ./Bundles/6.1-RELEASE_ubuntu_jammy_x86_64.artifactbundle.tar.gz + +``` + +This checksum should be provided along with the artifact bundles to be included with the sdk install invocation: + +```console +swift sdk install https://my-public-site/sdks/6.1-RELEASE_ubuntu_jammy_x86_64.artifactbundle.tar.gz --checksum +``` ## Contributing diff --git a/Sources/AsyncProcess/ChunkSequence.swift b/Sources/AsyncProcess/ChunkSequence.swift index 8daa1a3..fe60529 100644 --- a/Sources/AsyncProcess/ChunkSequence.swift +++ b/Sources/AsyncProcess/ChunkSequence.swift @@ -13,9 +13,9 @@ import NIO #if os(Linux) || os(Android) || os(Windows) -@preconcurrency import Foundation + @preconcurrency import Foundation #else -import Foundation + import Foundation #endif public struct IllegalStreamConsumptionError: Error { @@ -53,9 +53,9 @@ public struct ChunkSequence: AsyncSequence & Sendable { } else { throw IllegalStreamConsumptionError( description: """ - Either `.discard`ed, `.inherit`ed or redirected this stream to a `.fileHandle`, - cannot also consume it. To consume, please `.stream` it. - """ + Either `.discard`ed, `.inherit`ed or redirected this stream to a `.fileHandle`, + cannot also consume it. To consume, please `.stream` it. + """ ) } } diff --git a/Sources/AsyncProcess/FileContentStream.swift b/Sources/AsyncProcess/FileContentStream.swift index c704ea9..4999b8f 100644 --- a/Sources/AsyncProcess/FileContentStream.swift +++ b/Sources/AsyncProcess/FileContentStream.swift @@ -255,7 +255,10 @@ private final class ReadIntoAsyncChannelHandler: ChannelDuplexHandler { extension FileHandle { func fileContentStream(eventLoop: EventLoop) throws -> FileContentStream { - let asyncBytes = try FileContentStream(fileDescriptor: self.fileDescriptor, eventLoop: eventLoop) + let asyncBytes = try FileContentStream( + fileDescriptor: self.fileDescriptor, + eventLoop: eventLoop + ) try self.close() return asyncBytes } @@ -272,8 +275,8 @@ extension FileContentStream { } } -public extension AsyncSequence where Element == ByteBuffer, Self: Sendable { - func splitIntoLines( +extension AsyncSequence where Element == ByteBuffer, Self: Sendable { + public func splitIntoLines( dropTerminator: Bool = true, maximumAllowableBufferSize: Int = 1024 * 1024, dropLastChunkIfNoNewline: Bool = false @@ -286,14 +289,13 @@ public extension AsyncSequence where Element == ByteBuffer, Self: Sendable { ) } - var strings: AsyncMapSequence { + public var strings: AsyncMapSequence { self.map { String(buffer: $0) } } } public struct AsyncByteBufferLineSequence: AsyncSequence & Sendable - where Base: AsyncSequence, Base.Element == ByteBuffer -{ +where Base: AsyncSequence, Base.Element == ByteBuffer { public typealias Element = ByteBuffer private let underlying: Base private let dropTerminator: Bool @@ -329,7 +331,9 @@ public struct AsyncByteBufferLineSequence: AsyncSequence & Senda self.buffer.last?.readableBytesView } - mutating func concatenateEverything(upToLastChunkLengthToConsume lastLength: Int) -> ByteBuffer { + mutating func concatenateEverything(upToLastChunkLengthToConsume lastLength: Int) + -> ByteBuffer + { var output = ByteBuffer() output.reserveCapacity(lastLength + self.byteCountButLast) @@ -430,7 +434,8 @@ public struct AsyncByteBufferLineSequence: AsyncSequence & Senda } public init( - _ underlying: Base, dropTerminator: Bool, + _ underlying: Base, + dropTerminator: Bool, maximumAllowableBufferSize: Int, dropLastChunkIfNoNewline: Bool ) { diff --git a/Sources/AsyncProcess/ProcessExecutor+Convenience.swift b/Sources/AsyncProcess/ProcessExecutor+Convenience.swift index b7815cb..59e0302 100644 --- a/Sources/AsyncProcess/ProcessExecutor+Convenience.swift +++ b/Sources/AsyncProcess/ProcessExecutor+Convenience.swift @@ -53,7 +53,7 @@ public struct OutputLoggingSettings: Sendable { } } -public extension ProcessExecutor { +extension ProcessExecutor { /// Run child process, discarding all its output. /// /// - note: The `environment` defaults to the empty environment. @@ -69,7 +69,7 @@ public extension ProcessExecutor { /// don't want to /// provide input. /// - logger: Where to log diagnostic messages to (default to no where) - static func run( + public static func run( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -106,7 +106,7 @@ public extension ProcessExecutor { /// provide input. /// - logger: Where to log diagnostic and output messages to /// - logConfiguration: How to log the output lines - static func runLogOutput( + public static func runLogOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -128,7 +128,9 @@ public extension ProcessExecutor { return try await withThrowingTaskGroup(of: ProcessExitReason?.self) { group in group.addTask { for try await (stream, line) in await merge( - exe.standardOutput.splitIntoLines().strings.map { (ProcessOutputStream.standardOutput, $0) }, + exe.standardOutput.splitIntoLines().strings.map { + (ProcessOutputStream.standardOutput, $0) + }, exe.standardError.splitIntoLines().strings.map { (ProcessOutputStream.standardError, $0) } ) { logger.log( @@ -171,12 +173,12 @@ public extension ProcessExecutor { /// - splitOutputIntoLines: Whether to call the closure with full lines (`true`) or arbitrary chunks of output /// (`false`) /// - logger: Where to log diagnostic and output messages to - static func runProcessingOutput( + public static func runProcessingOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], standardInput: StandardInput, - outputProcessor: @escaping @Sendable (ProcessOutputStream, ByteBuffer) async throws -> (), + outputProcessor: @escaping @Sendable (ProcessOutputStream, ByteBuffer) async throws -> Void, splitOutputIntoLines: Bool = false, environment: [String: String] = [:], logger: Logger = ProcessExecutor.disableLogging @@ -225,11 +227,11 @@ public extension ProcessExecutor { } } - struct TooMuchProcessOutputError: Error, Sendable & Hashable { + public struct TooMuchProcessOutputError: Error, Sendable & Hashable { public var stream: ProcessOutputStream } - struct ProcessExitReasonAndOutput: Sendable & Hashable { + public struct ProcessExitReasonAndOutput: Sendable & Hashable { public var exitReason: ProcessExitReason public var standardOutput: ByteBuffer? public var standardError: ByteBuffer? @@ -260,7 +262,7 @@ public extension ProcessExecutor { /// - collectStandardError: If `true`, collect all of the child process' standard error into memory, discard if /// `false` /// - logger: Where to log diagnostic and output messages to - static func runCollectingOutput( + public static func runCollectingOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -287,7 +289,9 @@ public extension ProcessExecutor { if collectStandardOutput { var output: ByteBuffer? = nil for try await chunk in await exe.standardOutput { - guard (output?.readableBytes ?? 0) + chunk.readableBytes <= perStreamCollectionLimitBytes else { + guard + (output?.readableBytes ?? 0) + chunk.readableBytes <= perStreamCollectionLimitBytes + else { throw TooMuchProcessOutputError(stream: .standardOutput) } output.setOrWriteImmutableBuffer(chunk) @@ -302,7 +306,9 @@ public extension ProcessExecutor { if collectStandardError { var output: ByteBuffer? = nil for try await chunk in await exe.standardError { - guard (output?.readableBytes ?? 0) + chunk.readableBytes <= perStreamCollectionLimitBytes else { + guard + (output?.readableBytes ?? 0) + chunk.readableBytes <= perStreamCollectionLimitBytes + else { throw TooMuchProcessOutputError(stream: .standardError) } output.setOrWriteImmutableBuffer(chunk) @@ -317,7 +323,11 @@ public extension ProcessExecutor { try await .exitReason(exe.run()) } - var allInfo = ProcessExitReasonAndOutput(exitReason: .exit(-1), standardOutput: nil, standardError: nil) + var allInfo = ProcessExitReasonAndOutput( + exitReason: .exit(-1), + standardOutput: nil, + standardError: nil + ) while let next = try await group.next() { switch next { case let .exitReason(exitReason): @@ -333,7 +343,7 @@ public extension ProcessExecutor { } } -public extension ProcessExecutor { +extension ProcessExecutor { /// Run child process, discarding all its output. /// /// - note: The `environment` defaults to the empty environment. @@ -346,7 +356,7 @@ public extension ProcessExecutor { /// If you want to inherit the calling process' environment into the child, specify /// `ProcessInfo.processInfo.environment` /// - logger: Where to log diagnostic messages to (default to no where) - static func run( + public static func run( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -376,7 +386,7 @@ public extension ProcessExecutor { /// `ProcessInfo.processInfo.environment` /// - logger: Where to log diagnostic and output messages to /// - logConfiguration: How to log the output lines - static func runLogOutput( + public static func runLogOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -410,11 +420,11 @@ public extension ProcessExecutor { /// - splitOutputIntoLines: Whether to call the closure with full lines (`true`) or arbitrary chunks of output /// (`false`) /// - logger: Where to log diagnostic and output messages to - static func runProcessingOutput( + public static func runProcessingOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], - outputProcessor: @escaping @Sendable (ProcessOutputStream, ByteBuffer) async throws -> (), + outputProcessor: @escaping @Sendable (ProcessOutputStream, ByteBuffer) async throws -> Void, splitOutputIntoLines: Bool = false, environment: [String: String] = [:], logger: Logger = ProcessExecutor.disableLogging @@ -447,7 +457,7 @@ public extension ProcessExecutor { /// - collectStandardError: If `true`, collect all of the child process' standard error into memory, discard if /// `false` /// - logger: Where to log diagnostic and output messages to - static func runCollectingOutput( + public static func runCollectingOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -460,7 +470,8 @@ public extension ProcessExecutor { try await self.runCollectingOutput( group: group, executable: executable, - arguments, standardInput: EOFSequence(), + arguments, + standardInput: EOFSequence(), collectStandardOutput: collectStandardOutput, collectStandardError: collectStandardError, perStreamCollectionLimitBytes: perStreamCollectionLimitBytes, diff --git a/Sources/AsyncProcess/ProcessExecutor.swift b/Sources/AsyncProcess/ProcessExecutor.swift index 59f1d0a..4fe211d 100644 --- a/Sources/AsyncProcess/ProcessExecutor.swift +++ b/Sources/AsyncProcess/ProcessExecutor.swift @@ -11,13 +11,12 @@ //===----------------------------------------------------------------------===// import Atomics +import Foundation import Logging import NIO @_exported import struct SystemPackage.FileDescriptor -import Foundation - public struct ProcessOutputStream: Sendable & Hashable & CustomStringConvertible { enum Backing { case standardOutput @@ -159,7 +158,9 @@ public final actor ProcessExecutor { private let standardErrorWriteHandle: FileHandle? private let _standardOutput: ChunkSequence private let _standardError: ChunkSequence - private let processIsRunningApproximation = ManagedAtomic(RunningStateApproximation.neverStarted.rawValue) + private let processIsRunningApproximation = ManagedAtomic( + RunningStateApproximation.neverStarted.rawValue + ) private let processOutputConsumptionApproximation = ManagedAtomic(UInt8(0)) private let processPid = ManagedAtomic(pid_t(0)) private let ownsStandardOutputWriteHandle: Bool @@ -388,14 +389,14 @@ public final actor ProcessExecutor { { () -> Bool in guard outputConsumptionState.contains([.stdoutConsumed]) - || outputConsumptionState.contains([.stdoutNotStreamed]) + || outputConsumptionState.contains([.stdoutNotStreamed]) else { return false } guard outputConsumptionState.contains([.stderrConsumed]) - || outputConsumptionState.contains([.stderrNotStreamed]) + || outputConsumptionState.contains([.stderrNotStreamed]) else { return false } @@ -459,7 +460,7 @@ public final actor ProcessExecutor { case .processHasExited, .killedTheProcess: break loop case .processStillAlive: - () // gotta continue + () // gotta continue } } else { logger.debug("child process already dead") @@ -482,13 +483,13 @@ public final actor ProcessExecutor { public func run() async throws -> ProcessExitReason { let p = Process() #if canImport(Darwin) - if #available(macOS 13.0, *) { - p.executableURL = URL(filePath: self.executable) - } else { - p.launchPath = self.executable - } + if #available(macOS 13.0, *) { + p.executableURL = URL(filePath: self.executable) + } else { + p.launchPath = self.executable + } #else - p.executableURL = URL(fileURLWithPath: self.executable) + p.executableURL = URL(fileURLWithPath: self.executable) #endif p.arguments = self.arguments p.environment = self.environment @@ -559,15 +560,19 @@ public final actor ProcessExecutor { desired: RunningStateApproximation.finishedExecuting.rawValue, ordering: .relaxed ) - terminationStreamProducer.finish() // The termination handler will never have fired. - assert(worked) // We just set it to running above, shouldn't be able to race (no `await`). - assert(original == RunningStateApproximation.running.rawValue) // We compare-and-exchange it. + terminationStreamProducer.finish() // The termination handler will never have fired. + assert(worked) // We just set it to running above, shouldn't be able to race (no `await`). + assert(original == RunningStateApproximation.running.rawValue) // We compare-and-exchange it. throw error } // At this point, the process is running, we should therefore have a process ID (unless we're already dead). let childPid = p.processIdentifier - _ = self.processPid.compareExchange(expected: 0, desired: childPid, ordering: .sequentiallyConsistent) + _ = self.processPid.compareExchange( + expected: 0, + desired: childPid, + ordering: .sequentiallyConsistent + ) assert(childPid != 0 || !p.isRunning) self.logger.debug( "running command", @@ -578,12 +583,12 @@ public final actor ProcessExecutor { ] ) - try! self.standardInputPipe?.fileHandleForReading.close() // Must work. + try! self.standardInputPipe?.fileHandleForReading.close() // Must work. if self.ownsStandardOutputWriteHandle { - try! self.standardOutputWriteHandle?.close() // Must work. + try! self.standardOutputWriteHandle?.close() // Must work. } if self.ownsStandardErrorWriteHandle { - try! self.standardErrorWriteHandle?.close() // Must work. + try! self.standardErrorWriteHandle?.close() // Must work. } @Sendable @@ -611,9 +616,11 @@ public final actor ProcessExecutor { await withTaskGroup(of: Void.self) { triggerTeardownGroup in triggerTeardownGroup.addTask { // wait until cancelled - do { while true { - try await Task.sleep(nanoseconds: 1_000_000_000) - } } catch {} + do { + while true { + try await Task.sleep(nanoseconds: 1_000_000_000) + } + } catch {} let isRunning = self.processIsRunningApproximation.load(ordering: .relaxed) guard isRunning != RunningStateApproximation.finishedExecuting.rawValue else { @@ -639,7 +646,7 @@ public final actor ProcessExecutor { } let result = await waitForChildToExit() - triggerTeardownGroup.cancelAll() // This triggers the teardown + triggerTeardownGroup.cancelAll() // This triggers the teardown return result } } @@ -663,7 +670,7 @@ public final actor ProcessExecutor { exitReason = result } } - return exitReason! // must work because the real task will return a reason (or throw) + return exitReason! // must work because the real task will return a reason (or throw) } } @@ -679,21 +686,21 @@ public final actor ProcessExecutor { } } -public extension ProcessExecutor { +extension ProcessExecutor { /// A globally shared, singleton `EventLoopGroup` that's suitable for ``ProcessExecutor``. /// /// At present this is always `MultiThreadedEventLoopGroup.singleton`. - static var defaultEventLoopGroup: any EventLoopGroup { + public static var defaultEventLoopGroup: any EventLoopGroup { globalDefaultEventLoopGroup } /// The default `Logger` for ``ProcessExecutor`` that's used if you do not override it. It won't log anything. - static var disableLogging: Logger { + public static var disableLogging: Logger { globalDisableLoggingLogger } } -public extension ProcessExecutor { +extension ProcessExecutor { /// Create a ``ProcessExecutor`` to spawn a single child process. /// /// - note: The `environment` defaults to the empty environment. @@ -712,7 +719,7 @@ public extension ProcessExecutor { /// ``ProcessOutput/stream`` /// which requires to consume it via ``ProcessExecutor/standardError``. /// - logger: Where to log diagnostic messages to (default to no where) - init( + public init( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index a322f10..0724b1b 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -11,8 +11,10 @@ //===----------------------------------------------------------------------===// import ArgumentParser +import Foundation import Logging import SwiftSDKGenerator + import struct SystemPackage.FilePath @main @@ -54,10 +56,10 @@ struct GeneratorCLI: AsyncParsableCommand { } #if canImport(Darwin) - // On Darwin platforms Dispatch's signal source uses kqueue and EVFILT_SIGNAL for - // delivering signals. This exists alongside but with lower precedence than signal and - // sigaction: ignore signal handling here to kqueue can deliver signals. - signal(SIGINT, SIG_IGN) + // On Darwin platforms Dispatch's signal source uses kqueue and EVFILT_SIGNAL for + // delivering signals. This exists alongside but with lower precedence than signal and + // sigaction: ignore signal handling here to kqueue can deliver signals. + signal(SIGINT, SIG_IGN) #endif let signalSource = DispatchSource.makeSignalSource(signal: SIGINT) signalSource.setEventHandler { @@ -67,7 +69,10 @@ struct GeneratorCLI: AsyncParsableCommand { try await generatorTask.value } - logger.info("Generator run finished successfully.", metadata: ["elapsedTime": .string(elapsed.intervalString)]) + logger.info( + "Generator run finished successfully.", + metadata: ["elapsedTime": .string(elapsed.intervalString)] + ) } } @@ -85,14 +90,15 @@ extension GeneratorCLI { @Option( help: """ - Name of the SDK bundle. Defaults to a string composed of Swift version, Linux distribution, Linux release \ - and target CPU architecture. - """ + Name of the SDK bundle. Defaults to a string composed of Swift version, Linux distribution, Linux release \ + and target CPU architecture. + """ ) var sdkName: String? = nil @Flag( - help: "Experimental: avoid cleaning up toolchain and SDK directories and regenerate the SDK bundle incrementally." + help: + "Experimental: avoid cleaning up toolchain and SDK directories and regenerate the SDK bundle incrementally." ) var incremental: Bool = false @@ -101,51 +107,51 @@ extension GeneratorCLI { @Option( help: """ - Branch of Swift to use when downloading nightly snapshots. Specify `development` for snapshots off the `main` \ - branch of Swift open source project repositories. - """ + Branch of Swift to use when downloading nightly snapshots. Specify `development` for snapshots off the `main` \ + branch of Swift open source project repositories. + """ ) var swiftBranch: String? = nil @Option(help: "Version of Swift to supply in the bundle.") - var swiftVersion = "6.0.3-RELEASE" + var swiftVersion = "6.1-RELEASE" @Option( help: """ - Path to the Swift toolchain package containing the Swift compiler that runs on the host platform. - """ + Path to the Swift toolchain package containing the Swift compiler that runs on the host platform. + """ ) var hostSwiftPackagePath: String? = nil @Flag( inversion: .prefixedNo, help: """ - Whether or not to include the host toolchain in the Swift SDK. - If the host toolchain is not included, this makes the Swift SDK compatible with any host, \ - but requires exactly the same version of the swift.org toolchain to be installed for it to work. - """ + Whether or not to include the host toolchain in the Swift SDK. + If the host toolchain is not included, this makes the Swift SDK compatible with any host, \ + but requires exactly the same version of the swift.org toolchain to be installed for it to work. + """ ) var hostToolchain: Bool = false @Option( help: """ - Path to the Swift toolchain package containing the Swift standard library that runs on the target platform. - """ + Path to the Swift toolchain package containing the Swift standard library that runs on the target platform. + """ ) var targetSwiftPackagePath: String? = nil @Option( help: """ - The host triple of the bundle. Defaults to a triple of the machine this generator is \ - running on if unspecified. - """ + The host triple of the bundle. Defaults to a triple of the machine this generator is \ + running on if unspecified. + """ ) var host: Triple? = nil @Option( help: """ - The target triple of the bundle. The default depends on a recipe used for SDK generation. Pass `--help` to a specific recipe subcommand for more details. - """ + The target triple of the bundle. The default depends on a recipe used for SDK generation. Pass `--help` to a specific recipe subcommand for more details. + """ ) var target: Triple? = nil @@ -161,7 +167,9 @@ extension GeneratorCLI { let current = try SwiftSDKGenerator.getCurrentTriple(isVerbose: self.verbose) if let arch = hostArch { let target = Triple(arch: arch, vendor: current.vendor!, os: current.os!) - appLogger.warning("deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`") + appLogger.warning( + "deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`" + ) return target } return current @@ -173,8 +181,8 @@ extension GeneratorCLI { commandName: "make-linux-sdk", abstract: "Generate a Swift SDK bundle for Linux.", discussion: """ - The default `--target` triple is Linux with the same CPU architecture with host triple - """ + The default `--target` triple is Linux with the same CPU architecture with host triple + """ ) @OptionGroup @@ -191,18 +199,20 @@ extension GeneratorCLI { @Option( help: """ - Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`. - """, + Linux distribution to use if the target platform is Linux. + - Available options: `ubuntu`, `debian`, `rhel`. Default is `ubuntu`. + """, transform: LinuxDistribution.Name.init(nameString:) ) var linuxDistributionName = LinuxDistribution.Name.ubuntu @Option( help: """ - Version of the Linux distribution used as a target platform. - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). - """ + Version of the Linux distribution used as a target platform. + - Available options for Ubuntu: `20.04`, `22.04` (default when `--distribution-name` is `ubuntu`), `24.04`. + - Available options for Debian: `11`, `12` (default when `--distribution-name` is `debian`). + - Available options for RHEL: `ubi9` (default when `--distribution-name` is `rhel`). + """ ) var linuxDistributionVersion: String? @@ -212,7 +222,9 @@ extension GeneratorCLI { } if let arch = generatorOptions.targetArch { let target = Triple(arch: arch, vendor: nil, os: .linux, environment: .gnu) - appLogger.warning("deprecated: Please use `--target \(target.triple)` instead of `--target-arch \(arch)`") + appLogger.warning( + "deprecated: Please use `--target \(target.triple)` instead of `--target-arch \(arch)`" + ) } return Triple(arch: hostTriple.arch!, vendor: nil, os: .linux, environment: .gnu) } @@ -229,9 +241,15 @@ extension GeneratorCLI { linuxDistributionDefaultVersion = "ubi9" case .ubuntu: linuxDistributionDefaultVersion = "22.04" + case .debian: + linuxDistributionDefaultVersion = "12" } - let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion - let linuxDistribution = try LinuxDistribution(name: linuxDistributionName, version: linuxDistributionVersion) + let linuxDistributionVersion = + self.linuxDistributionVersion ?? linuxDistributionDefaultVersion + let linuxDistribution = try LinuxDistribution( + name: linuxDistributionName, + version: linuxDistributionVersion + ) let hostTriple = try self.generatorOptions.deriveHostTriple() let targetTriple = self.deriveTargetTriple(hostTriple: hostTriple) @@ -249,7 +267,11 @@ extension GeneratorCLI { includeHostToolchain: self.generatorOptions.hostToolchain, logger: loggerWithLevel(from: self.generatorOptions) ) - try await GeneratorCLI.run(recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions) + try await GeneratorCLI.run( + recipe: recipe, + targetTriple: targetTriple, + options: self.generatorOptions + ) } func isInvokedAsDefaultSubcommand() -> Bool { @@ -272,8 +294,8 @@ extension GeneratorCLI { commandName: "make-wasm-sdk", abstract: "Experimental: Generate a Swift SDK bundle for WebAssembly.", discussion: """ - The default `--target` triple is wasm32-unknown-wasi - """ + The default `--target` triple is wasm32-unknown-wasi + """ ) @OptionGroup @@ -281,8 +303,8 @@ extension GeneratorCLI { @Option( help: """ - Path to the WASI sysroot directory containing the WASI libc headers and libraries. - """ + Path to the WASI sysroot directory containing the WASI libc headers and libraries. + """ ) var wasiSysroot: String @@ -305,32 +327,44 @@ extension GeneratorCLI { logger: loggerWithLevel(from: self.generatorOptions) ) let targetTriple = self.deriveTargetTriple() - try await GeneratorCLI.run(recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions) + try await GeneratorCLI.run( + recipe: recipe, + targetTriple: targetTriple, + options: self.generatorOptions + ) } } } -import Foundation - extension Duration { var intervalString: String { let reference = Date() let date = Date(timeInterval: TimeInterval(self.components.seconds), since: reference) - let components = Calendar.current.dateComponents([.hour, .minute, .second], from: reference, to: date) + let components = Calendar.current.dateComponents( + [.hour, .minute, .second], + from: reference, + to: date + ) if let hours = components.hour, hours > 0 { #if !canImport(Darwin) && compiler(<6.0) - return String(format: "%02d:%02d:%02d", hours, components.minute ?? 0, components.second ?? 0) + return String( + format: "%02d:%02d:%02d", + hours, + components.minute ?? 0, + components.second ?? 0 + ) #else - return self.formatted() + return self.formatted() #endif } else if let minutes = components.minute, minutes > 0 { #if !canImport(Darwin) && compiler(<6.0) - let seconds = components.second ?? 0 - return "\(minutes) minute\(minutes != 1 ? "s" : "") \(seconds) second\(seconds != 1 ? "s" : "")" + let seconds = components.second ?? 0 + return + "\(minutes) minute\(minutes != 1 ? "s" : "") \(seconds) second\(seconds != 1 ? "s" : "")" #else - return "\(self.formatted(.time(pattern: .minuteSecond))) seconds" + return "\(self.formatted(.time(pattern: .minuteSecond))) seconds" #endif } else { return "\(components.second ?? 0) seconds" diff --git a/Sources/Helpers/ThrowingDefer.swift b/Sources/Helpers/ThrowingDefer.swift index 9fec320..899c581 100644 --- a/Sources/Helpers/ThrowingDefer.swift +++ b/Sources/Helpers/ThrowingDefer.swift @@ -22,7 +22,7 @@ /// - SeeAlso: ``withAsyncThrowing(do:defer:)`` public func withThrowing( do work: () throws -> T, - defer deferred: () throws -> () + defer deferred: () throws -> Void ) throws -> T { do { let result = try work() @@ -46,7 +46,7 @@ public func withThrowing( /// - SeeAlso: ``withThrowing(do:defer:)`` public func withAsyncThrowing( do work: @Sendable () async throws -> T, - defer deferred: @Sendable () async throws -> () + defer deferred: @Sendable () async throws -> Void ) async throws -> T { do { let result = try await work() diff --git a/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift index 6904828..ac961ca 100644 --- a/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift +++ b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift @@ -182,4 +182,3 @@ extension Array: CacheKey where Element == FilePath.Component { map(\.string).joined(separator: "\n").hash(with: &hashFunction) } } - diff --git a/Sources/Helpers/Vendor/Triple.swift b/Sources/Helpers/Vendor/Triple.swift index fd256d4..0fe731e 100644 --- a/Sources/Helpers/Vendor/Triple.swift +++ b/Sources/Helpers/Vendor/Triple.swift @@ -1,6 +1,6 @@ -//===--------------- Triple.swift - Swift Target Triples ------------------===// +//===----------------------------------------------------------------------===// // -// This source file is part of the Swift.org open source project +// This source file is part of the Swift open source project // // Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift index b62b190..50d3c54 100644 --- a/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift @@ -1,4 +1,3 @@ - //===----------------------------------------------------------------------===// // // This source file is part of the Swift open source project @@ -6,8 +5,8 @@ // Copyright (c) 2020-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // //===----------------------------------------------------------------------===// diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift index 844ba04..dd5657a 100644 --- a/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift @@ -1,12 +1,14 @@ -/* - This source file is part of the Swift.org open source project - - Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors - Licensed under Apache License v2.0 with Runtime Library Exception - - See http://swift.org/LICENSE.txt for license information - See http://swift.org/CONTRIBUTORS.txt for Swift project authors -*/ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// import _Concurrency diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index 88fb7d5..1af7fd1 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -10,12 +10,13 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.URL import Helpers + +import struct Foundation.URL import struct SystemPackage.FilePath -private extension Triple { - var llvmBinaryURLSuffix: String { +extension Triple { + fileprivate var llvmBinaryURLSuffix: String { switch (self.os, self.arch) { case (.linux, .aarch64): return "aarch64-linux-gnu" case (.linux, .x86_64): return "x86_64-linux-gnu-ubuntu-22.04" @@ -53,11 +54,12 @@ struct DownloadableArtifacts: Sendable { if hostTriple.os == .linux { // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts - let linuxArchSuffix = hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + let hostArchSuffix = + hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" self.hostSwift = .init( remoteURL: versions.swiftDownloadURL( - subdirectory: "amazonlinux2\(linuxArchSuffix)", - platform: "amazonlinux2\(linuxArchSuffix)", + subdirectory: "amazonlinux2\(hostArchSuffix)", + platform: "amazonlinux2\(hostArchSuffix)", fileExtension: "tar.gz" ), localPath: paths.artifactsCachePath @@ -80,12 +82,12 @@ struct DownloadableArtifacts: Sendable { self.hostLLVM = .init( remoteURL: URL( string: """ - https://github.com/llvm/llvm-project/releases/download/llvmorg-\( - versions.lldVersion - )/clang+llvm-\( - versions.lldVersion - )-\(hostTriple.llvmBinaryURLSuffix).tar.xz - """ + https://github.com/llvm/llvm-project/releases/download/llvmorg-\( + versions.lldVersion + )/clang+llvm-\( + versions.lldVersion + )-\(hostTriple.llvmBinaryURLSuffix).tar.xz + """ )!, localPath: paths.artifactsCachePath .appending("host_llvm_\(versions.lldVersion)_\(hostTriple.triple).tar.xz"), @@ -95,7 +97,9 @@ struct DownloadableArtifacts: Sendable { self.targetSwift = .init( remoteURL: versions.swiftDownloadURL(), localPath: paths.artifactsCachePath - .appending("target_swift_\(versions.swiftVersion)_\(targetTriple.triple).tar.gz"), + .appending( + "target_swift_\(versions.swiftVersion)_\(versions.swiftPlatform)_\(targetTriple.archName).tar.gz" + ), isPrebuilt: true ) } @@ -104,12 +108,12 @@ struct DownloadableArtifacts: Sendable { self.hostLLVM = .init( remoteURL: URL( string: """ - https://github.com/llvm/llvm-project/releases/download/llvmorg-\( - self.versions.lldVersion - )/llvm-project-\( - self.versions.lldVersion - ).src.tar.xz - """ + https://github.com/llvm/llvm-project/releases/download/llvmorg-\( + self.versions.lldVersion + )/llvm-project-\( + self.versions.lldVersion + ).src.tar.xz + """ )!, localPath: self.paths.artifactsCachePath .appending("llvm_\(self.versions.lldVersion).src.tar.xz"), diff --git a/Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift b/Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift index 139e371..0ad671b 100644 --- a/Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift +++ b/Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift @@ -1,10 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + extension String { - func hasAnyPrefix(from array: [String]) -> Bool { - for item in array { - if self.hasPrefix(item) { - return true - } - } - return false + func hasAnyPrefix(from array: [String]) -> Bool { + for item in array { + if self.hasPrefix(item) { + return true + } } + return false + } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 5b9afae..b6d9d7c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -import SystemPackage import Foundation +import SystemPackage extension SwiftSDKGenerator { func copyTargetSwiftFromDocker( @@ -22,6 +22,7 @@ extension SwiftSDKGenerator { logger.info("Launching a container to extract the Swift SDK for the target triple...") try await withDockerContainer(fromImage: baseDockerImage) { containerID in try await inTemporaryDirectory { generator, _ in + let sdkLibPath = sdkDirPath.appending("lib") let sdkUsrPath = sdkDirPath.appending("usr") try await generator.createDirectoryIfNeeded(at: sdkUsrPath) try await generator.copyFromDockerContainer( @@ -34,23 +35,23 @@ extension SwiftSDKGenerator { try await generator.runOnDockerContainer( id: containerID, command: #""" - sh -c ' - chmod +w /usr/lib64 - cd /usr/lib64 - for n in *; do - destination=$(readlink $n) - echo $destination | grep "\.\." && \ - rm -f $n && \ - ln -s $(basename $destination) $n - done - rm -rf pm-utils - ' - """# + sh -c ' + chmod +w /usr/lib64 + cd /usr/lib64 + for n in *; do + destination=$(readlink $n) + echo $destination | grep "\.\." && \ + rm -f $n && \ + ln -s $(basename $destination) $n + done + rm -rf pm-utils + ' + """# ) } if case let containerLib64 = FilePath("/usr/lib64"), - try await generator.doesPathExist(containerLib64, inContainer: containerID) + try await generator.doesPathExist(containerLib64, inContainer: containerID) { let sdkUsrLib64Path = sdkUsrPath.appending("lib64") // we already checked that the path exists above, so we don't pass `failIfNotExists: false` here. @@ -60,6 +61,15 @@ extension SwiftSDKGenerator { to: sdkUsrLib64Path ) try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") + } else if case let containerLib64 = FilePath("/lib64"), + try await generator.doesPathExist(containerLib64, inContainer: containerID) + { + let sdkLib64Path = sdkDirPath.appending("lib64") + try await generator.copyFromDockerContainer( + id: containerID, + from: containerLib64, + to: sdkLib64Path + ) } let sdkUsrLibPath = sdkUsrPath.appending("lib") @@ -72,13 +82,26 @@ extension SwiftSDKGenerator { // architecture-specific directories: // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. - if case .ubuntu = targetDistribution { - subpaths += [("\(targetTriple.archName)-linux-gnu", false)] + if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { + var archSubpath = "\(targetTriple.archName)-linux-gnu" - // Custom subpath for armv7 + // armv7 with Debian uses a custom subpath for armhf if targetTriple.archName == "armv7" { - subpaths += [("arm-linux-gnueabihf", false)] + archSubpath = "arm-linux-gnueabihf" } + + // Copy /lib/ for Debian 11 + if case let .debian(debian) = targetDistribution, debian == .bullseye { + try await generator.createDirectoryIfNeeded(at: sdkLibPath) + try await generator.copyFromDockerContainer( + id: containerID, + from: FilePath("/lib").appending(archSubpath), + to: sdkLibPath.appending(archSubpath), + failIfNotExists: false + ) + } + + subpaths += [(archSubpath, false)] } for (subpath, failIfNotExists) in subpaths { @@ -89,14 +112,33 @@ extension SwiftSDKGenerator { failIfNotExists: failIfNotExists ) } - try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib") + + // Symlink if we do not have a /lib directory in the SDK + if await !generator.doesFileExist(at: sdkLibPath) { + try await generator.createSymlink(at: sdkLibPath, pointingTo: "usr/lib") + } + + // Look for 32-bit libraries to remove from RHEL-based distros + // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up + if case .rhel = targetDistribution { + for gccVersion in 7...13 { + let removePath = "gcc/x86_64-redhat-linux/\(gccVersion)/32" + if await doesFileExist(at: sdkUsrLibPath.appending(removePath)) { + logger.warning( + "Removing 32-bit libraries from RHEL imported sysroot", + metadata: ["removePath": .stringConvertible(removePath)] + ) + try await removeRecursively(at: sdkUsrLibPath.appending(removePath)) + } + } + } // Copy the ELF interpreter try await generator.copyFromDockerContainer( - id: containerID, - from: FilePath(targetTriple.interpreterPath), - to: sdkDirPath.appending(targetTriple.interpreterPath), - failIfNotExists: true + id: containerID, + from: FilePath(targetTriple.interpreterPath), + to: sdkDirPath.appending(targetTriple.interpreterPath), + failIfNotExists: true ) // Python artifacts are redundant. @@ -119,7 +161,10 @@ extension SwiftSDKGenerator { let fromPath = distributionPath.appending(pathWithinPackage) if isOptional && !doesFileExist(at: fromPath) { - logger.debug("Optional package path ignored since it does not exist", metadata: ["packagePath": .string(fromPath.string)]) + logger.debug( + "Optional package path ignored since it does not exist", + metadata: ["packagePath": .string(fromPath.string)] + ) continue } @@ -131,10 +176,10 @@ extension SwiftSDKGenerator { extension Triple { var interpreterPath: String { switch self.archName { - case "x86_64": return "/lib64/ld-linux-x86-64.so.2" - case "aarch64": return "/lib/ld-linux-aarch64.so.1" - case "armv7": return "/lib/ld-linux-armhf.so.3" - default: fatalError("unsupported architecture \(self.archName)") + case "x86_64": return "/lib64/ld-linux-x86-64.so.2" + case "aarch64": return "/lib/ld-linux-aarch64.so.1" + case "armv7": return "/lib/ld-linux-armhf.so.3" + default: fatalError("unsupported architecture \(self.archName)") } } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 59db653..d7f1935 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -11,18 +11,18 @@ //===----------------------------------------------------------------------===// import AsyncAlgorithms -import Logging import Helpers +import Logging import RegexBuilder import class Foundation.ByteCountFormatter import class Foundation.FileManager import struct Foundation.URL - import struct SystemPackage.FilePath -private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" -private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" +private let ubuntuMainMirror = "http://gb.archive.ubuntu.com/ubuntu" +private let ubuntuPortsMirror = "http://ports.ubuntu.com/ubuntu-ports" +private let debianMirror = "http://deb.debian.org/debian" extension FilePath { var metadataValue: Logger.MetadataValue { @@ -32,7 +32,8 @@ extension FilePath { extension SwiftSDKGenerator { func downloadArtifacts( - _ client: some HTTPClientProtocol, _ engine: QueryEngine, + _ client: some HTTPClientProtocol, + _ engine: QueryEngine, downloadableArtifacts: inout DownloadableArtifacts, itemsToDownload: @Sendable (DownloadableArtifacts) -> [DownloadableArtifacts.Item] ) async throws { @@ -40,7 +41,9 @@ extension SwiftSDKGenerator { let hostLLVMURL = downloadableArtifacts.hostLLVM.remoteURL // Workaround an issue with github.com returning 400 instead of 404 status to HEAD requests from AHC. - if itemsToDownload(downloadableArtifacts).contains(where: { $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL } ) { + if itemsToDownload(downloadableArtifacts).contains(where: { + $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL + }) { let isLLVMBinaryArtifactAvailable = try await type(of: client).with(http1Only: true) { try await $0.head( url: hostLLVMURL.absoluteString, @@ -56,7 +59,9 @@ extension SwiftSDKGenerator { let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in for item in itemsToDownload(downloadableArtifacts) { group.addTask { - try await engine[DownloadArtifactQuery(artifact: item, httpClient: client, logger: self.logger)] + try await engine[ + DownloadArtifactQuery(artifact: item, httpClient: client, logger: self.logger) + ] } } @@ -68,69 +73,131 @@ extension SwiftSDKGenerator { } logger.info("Using downloaded artifacts from cache") - logger.debug("Using downloaded artifacts in these locations.", metadata: [ - "paths": .array(results.map(\.path.metadataValue)) - ]) + logger.debug( + "Using downloaded artifacts in these locations.", + metadata: [ + "paths": .array(results.map(\.path.metadataValue)) + ] + ) } - func downloadUbuntuPackages( + func getMirrorURL(for linuxDistribution: LinuxDistribution) throws -> String { + if linuxDistribution.name == .ubuntu { + if targetTriple.arch == .x86_64 { + return ubuntuMainMirror + } else { + return ubuntuPortsMirror + } + } else if linuxDistribution.name == .debian { + return debianMirror + } else { + throw GeneratorError.distributionSupportsOnlyDockerGenerator(linuxDistribution) + } + } + + func packagesFileName(isXzAvailable: Bool) -> String { + if isXzAvailable { + return "Packages.xz" + } + // Use .gz if xz is not available + return "Packages.gz" + } + + func downloadDebianPackages( _ client: some HTTPClientProtocol, _ engine: QueryEngine, requiredPackages: [String], versionsConfiguration: VersionsConfiguration, sdkDirPath: FilePath ) async throws { - logger.debug("Parsing Ubuntu packages list...") + let mirrorURL = try getMirrorURL(for: versionsConfiguration.linuxDistribution) + let distributionName = versionsConfiguration.linuxDistribution.name + let distributionRelease = versionsConfiguration.linuxDistribution.release // Find xz path let xzPath = try await which("xz") if xzPath == nil { - logger.warning(""" - The `xz` utility was not found in `PATH`. \ - Consider installing it for more efficient downloading of package lists. - """) - } + // If we don't have xz, it's required for Packages.xz for debian + if distributionName == .debian { + throw GeneratorError.debianPackagesListDownloadRequiresXz + } - async let mainPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) + logger.warning( + """ + The `xz` utility was not found in `PATH`. \ + Consider installing it for more efficient downloading of package lists. + """ + ) + } - async let updatesPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath + logger.info( + "Downloading and parsing packages lists...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "distributionRelease": .string(distributionRelease), + ] ) - async let universePackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "universe", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) + let allPackages = try await withThrowingTaskGroup(of: [String: URL].self) { group in + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + if distributionName == .ubuntu { + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "universe", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + } - let allPackages = try await mainPackages - .merging(updatesPackages, uniquingKeysWith: { $1 }) - .merging(universePackages, uniquingKeysWith: { $1 }) + var packages: [String: URL] = [String: URL]() + for try await result in group { + packages.merge(result, uniquingKeysWith: { $1 }) + } + return packages + } let urls = requiredPackages.compactMap { allPackages[$0] } guard urls.count == requiredPackages.count else { - throw GeneratorError.ubuntuPackagesParsingFailure( + throw GeneratorError.packagesListParsingFailure( expectedPackages: requiredPackages.count, actual: urls.count ) } - logger.info("Downloading Ubuntu packages...", metadata: ["packageCount": .stringConvertible(urls.count)]) + logger.info( + "Downloading packages...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "packageCount": .stringConvertible(urls.count), + ] + ) try await inTemporaryDirectory { fs, tmpDir in let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine) await report(downloadedFiles: downloadedFiles) @@ -140,98 +207,46 @@ extension SwiftSDKGenerator { try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath) } } - } - func downloadFiles( - from urls: [URL], - to directory: FilePath, - _ client: some HTTPClientProtocol, - _ engine: QueryEngine - ) async throws -> [(URL, UInt64)] { - try await withThrowingTaskGroup(of: (URL, UInt64).self) { - for url in urls { - $0.addTask { - let downloadedFilePath = try await engine[DownloadFileQuery( - remoteURL: url, localDirectory: directory, httpClient: client - )] - let filePath = downloadedFilePath.path - guard let fileSize = try FileManager.default.attributesOfItem( - atPath: filePath.string - )[.size] as? UInt64 else { - throw GeneratorError.fileDoesNotExist(filePath) - } - return (url, fileSize) - } + // Make sure we have /lib and /lib64, and if not symlink from /usr + // This makes building from packages more consistent with copying from the Docker container + let libDirectories = ["lib", "lib64"] + for dir in libDirectories { + let sdkLibPath = sdkDirPath.appending(dir) + let sdkUsrLibPath = sdkDirPath.appending("usr/\(dir)") + if !doesFileExist(at: sdkLibPath) && doesFileExist(at: sdkUsrLibPath) { + try createSymlink(at: sdkLibPath, pointingTo: FilePath("./usr/\(dir)")) } - - var result = [(URL, UInt64)]() - for try await progress in $0 { - result.append(progress) - } - return result - } - } - - private func report(downloadedFiles: [(URL, UInt64)]) { - let byteCountFormatter = ByteCountFormatter() - - for (url, bytes) in downloadedFiles { - logger.debug("Downloaded package", metadata: [ - "url": .string(url.absoluteString), - "size": .string(byteCountFormatter.string(fromByteCount: Int64(bytes))) - ]) - } - } -} - -extension HTTPClientProtocol { - private func downloadUbuntuPackagesList( - from url: String, - unzipWith zipPath: String, - isVerbose: Bool - ) async throws -> String? { - guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) else { - throw FileOperationError.downloadFailed(url) } - - return String(buffer: packages) } - func packagesFileName(isXzAvailable: Bool) -> String { - if isXzAvailable { - return "Packages.xz" - } - // Use .gz if xz is not available - return "Packages.gz" - } - - func parseUbuntuPackagesList( - ubuntuRelease: String, - releaseSuffix: String = "", + private func parseDebianPackageList( + using client: HTTPClientProtocol, + mirrorURL: String, + release: String, + releaseSuffix: String, repository: String, targetTriple: Triple, - isVerbose: Bool, xzPath: String? ) async throws -> [String: URL] { - let mirrorURL: String - if targetTriple.arch == .x86_64 { - mirrorURL = ubuntuAMD64Mirror - } else { - mirrorURL = ubuntuARM64Mirror - } + var contextLogger = logger let packagesListURL = """ - \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\( + \(mirrorURL)/dists/\(release)\(releaseSuffix)/\(repository)/binary-\( targetTriple.arch!.debianConventionName - )/\(packagesFileName(isXzAvailable: xzPath != nil)) - """ - - guard let packages = try await downloadUbuntuPackagesList( - from: packagesListURL, - unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available - isVerbose: isVerbose - ) else { - throw GeneratorError.ubuntuPackagesDecompressionFailure + )/\(packagesFileName(isXzAvailable: xzPath != nil)) + """ + contextLogger[metadataKey: "packagesListURL"] = .string(packagesListURL) + + contextLogger.debug("Downloading packages list...") + guard + let packages = try await client.downloadDebianPackagesList( + from: packagesListURL, + unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available + logger: logger + ) + else { + throw GeneratorError.packagesListDecompressionFailure } let packageRef = Reference(Substring.self) @@ -253,14 +268,9 @@ extension HTTPClientProtocol { } Anchor.endOfLine - - OneOrMore(.any, .reluctant) - - "Description-md5: " - - OneOrMore(.hexDigit) } + contextLogger.debug("Processing packages list...") var result = [String: URL]() for match in packages.matches(of: regex) { guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } @@ -270,4 +280,69 @@ extension HTTPClientProtocol { return result } + + func downloadFiles( + from urls: [URL], + to directory: FilePath, + _ client: some HTTPClientProtocol, + _ engine: QueryEngine + ) async throws -> [(URL, UInt64)] { + try await withThrowingTaskGroup(of: (URL, UInt64).self) { + for url in urls { + $0.addTask { + let downloadedFilePath = try await engine[ + DownloadFileQuery( + remoteURL: url, + localDirectory: directory, + httpClient: client + ) + ] + let filePath = downloadedFilePath.path + guard + let fileSize = try FileManager.default.attributesOfItem( + atPath: filePath.string + )[.size] as? UInt64 + else { + throw GeneratorError.fileDoesNotExist(filePath) + } + return (url, fileSize) + } + } + + var result = [(URL, UInt64)]() + for try await progress in $0 { + result.append(progress) + } + return result + } + } + + private func report(downloadedFiles: [(URL, UInt64)]) { + let byteCountFormatter = ByteCountFormatter() + + for (url, bytes) in downloadedFiles { + logger.debug( + "Downloaded package", + metadata: [ + "url": .string(url.absoluteString), + "size": .string(byteCountFormatter.string(fromByteCount: Int64(bytes))), + ] + ) + } + } +} + +extension HTTPClientProtocol { + func downloadDebianPackagesList( + from url: String, + unzipWith zipPath: String, + logger: Logger + ) async throws -> String? { + guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, logger: logger) + else { + throw FileOperationError.downloadFailed(url) + } + + return String(buffer: packages) + } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index 8bf18f5..3e8d550 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -11,17 +11,18 @@ //===----------------------------------------------------------------------===// import AsyncAlgorithms -#if canImport(AsyncHTTPClient) -import AsyncHTTPClient -#endif import Foundation import Helpers import RegexBuilder import SystemPackage -public extension Triple.Arch { +#if canImport(AsyncHTTPClient) + import AsyncHTTPClient +#endif + +extension Triple.Arch { /// Returns the value of `cpu` converted to a convention used in Debian package names - var debianConventionName: String { + public var debianConventionName: String { switch self { case .aarch64: return "arm64" case .x86_64: return "amd64" @@ -32,14 +33,15 @@ public extension Triple.Arch { } } -public extension SwiftSDKGenerator { - func run(recipe: SwiftSDKRecipe) async throws { - try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in +extension SwiftSDKGenerator { + package func run(recipe: SwiftSDKRecipe) async throws { + try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { + engine in let httpClientType: HTTPClientProtocol.Type #if canImport(AsyncHTTPClient) - httpClientType = HTTPClient.self + httpClientType = HTTPClient.self #else - httpClientType = OfflineHTTPClient.self + httpClientType = OfflineHTTPClient.self #endif try await httpClientType.with { client in if !self.isIncremental { @@ -48,27 +50,53 @@ public extension SwiftSDKGenerator { try await self.createDirectoryIfNeeded(at: pathsConfiguration.artifactsCachePath) - let swiftSDKProduct = try await recipe.makeSwiftSDK(generator: self, engine: engine, httpClient: client) + let swiftSDKProduct = try await recipe.makeSwiftSDK( + generator: self, + engine: engine, + httpClient: client + ) let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe) - try await generateDestinationJSON( - toolsetPath: toolsetJSONPath, - sdkDirPath: swiftSDKProduct.sdkDirPath, - recipe: recipe + var artifacts = try await [ + self.artifactID: generateSwiftSDKMetadata( + toolsetPath: toolsetJSONPath, + sdkDirPath: swiftSDKProduct.sdkDirPath, + recipe: recipe + ) + ] + + if recipe.shouldSupportEmbeddedSwift { + let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe, isForEmbeddedSwift: true) + + artifacts["\(self.artifactID)-embedded"] = try await generateSwiftSDKMetadata( + toolsetPath: toolsetJSONPath, + sdkDirPath: swiftSDKProduct.sdkDirPath, + recipe: recipe, + isForEmbeddedSwift: true + ) + } + + try await generateArtifactBundleManifest( + hostTriples: swiftSDKProduct.hostTriples, + artifacts: artifacts ) - try await generateArtifactBundleManifest(hostTriples: swiftSDKProduct.hostTriples) + #if compiler(>=6.0) + let sdkCommandPrefix = "" + #else + let sdkCommandPrefix = "experimental-" + #endif // Extra spaces added for readability for the user print( """ All done! Install the newly generated SDK with this command: - swift experimental-sdk install \(pathsConfiguration.artifactBundlePath) + swift \(sdkCommandPrefix)sdk install \(pathsConfiguration.artifactBundlePath) After that, use the newly installed SDK when building with this command: - swift build --experimental-swift-sdk \(artifactID) + swift build --\(sdkCommandPrefix)swift-sdk \(artifactID) """ ) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift index ea587cb..493db05 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift @@ -16,6 +16,14 @@ import SystemPackage import struct Foundation.Data extension SwiftSDKGenerator { + func createLibSymlink(sdkDirPath: FilePath) throws { + let libPath = sdkDirPath.appending("lib") + if !doesFileExist(at: libPath) { + logger.info("Adding lib symlink to usr/lib...") + try createSymlink(at: libPath, pointingTo: "usr/lib") + } + } + func fixAbsoluteSymlinks(sdkDirPath: FilePath) throws { logger.info("Fixing up absolute symlinks...") @@ -49,7 +57,9 @@ extension SwiftSDKGenerator { } func symlinkClangHeaders() throws { - let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang") + let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending( + "usr/lib/swift_static/clang" + ) if !doesFileExist(at: swiftStaticClangPath) { logger.info("Symlinking clang headers...") try self.createSymlink( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index 8388617..fbb056b 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -21,10 +21,12 @@ private let encoder: JSONEncoder = { }() extension SwiftSDKGenerator { - func generateToolsetJSON(recipe: SwiftSDKRecipe) throws -> FilePath { + func generateToolsetJSON(recipe: SwiftSDKRecipe, isForEmbeddedSwift: Bool = false) throws -> FilePath { logger.info("Generating toolset JSON file...") - let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending("toolset.json") + let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending( + "\(isForEmbeddedSwift ? "embedded-" : "")toolset.json" + ) var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath @@ -37,16 +39,27 @@ extension SwiftSDKGenerator { } var toolset = Toolset(rootPath: relativeToolchainBinDir.string) - recipe.applyPlatformOptions(toolset: &toolset, targetTriple: self.targetTriple) + recipe.applyPlatformOptions( + toolset: &toolset, + targetTriple: self.targetTriple, + isForEmbeddedSwift: isForEmbeddedSwift + ) try writeFile(at: toolsetJSONPath, encoder.encode(toolset)) return toolsetJSONPath } - func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath, recipe: SwiftSDKRecipe) throws { - logger.info("Generating destination JSON file...") + func generateSwiftSDKMetadata( + toolsetPath: FilePath, + sdkDirPath: FilePath, + recipe: SwiftSDKRecipe, + isForEmbeddedSwift: Bool = false + ) throws -> FilePath { + logger.info("Generating Swift SDK metadata JSON file...") - let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending("swift-sdk.json") + let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending( + "\(isForEmbeddedSwift ? "embedded-" : "")swift-sdk.json" + ) var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath var relativeSDKDir = sdkDirPath @@ -57,36 +70,39 @@ extension SwiftSDKGenerator { relativeSDKDir.removePrefix(pathsConfiguration.swiftSDKRootPath), relativeToolsetPath.removePrefix(pathsConfiguration.swiftSDKRootPath) else { - fatalError(""" - `toolchainBinDirPath`, `sdkDirPath`, and `toolsetPath` are at unexpected locations that prevent computing \ - relative paths - """) + fatalError( + """ + `toolchainBinDirPath`, `sdkDirPath`, and `toolsetPath` are at unexpected locations that prevent computing \ + relative paths + """ + ) } - var metadata = SwiftSDKMetadataV4.TripleProperties( - sdkRootPath: relativeSDKDir.string, - toolsetPaths: [relativeToolsetPath.string] + var metadata = SwiftSDKMetadataV4( + targetTriples: [ + self.targetTriple.triple: .init( + sdkRootPath: relativeSDKDir.string, + toolsetPaths: [relativeToolsetPath.string] + ) + ] ) recipe.applyPlatformOptions( metadata: &metadata, paths: pathsConfiguration, - targetTriple: self.targetTriple + targetTriple: self.targetTriple, + isForEmbeddedSwift: isForEmbeddedSwift ) try writeFile( at: destinationJSONPath, - encoder.encode( - SwiftSDKMetadataV4( - targetTriples: [ - self.targetTriple.triple: metadata, - ] - ) - ) + encoder.encode(metadata) ) + + return destinationJSONPath } - func generateArtifactBundleManifest(hostTriples: [Triple]?) throws { + func generateArtifactBundleManifest(hostTriples: [Triple]?, artifacts: [String: FilePath]) throws { logger.info("Generating .artifactbundle info JSON file...") let artifactBundleManifestPath = pathsConfiguration.artifactBundlePath.appending("info.json") @@ -96,18 +112,22 @@ extension SwiftSDKGenerator { encoder.encode( ArtifactsArchiveMetadata( schemaVersion: "1.0", - artifacts: [ - artifactID: .init( + artifacts: artifacts.mapValues { + var relativePath = $0 + let prefixRemoved = relativePath.removePrefix(pathsConfiguration.artifactBundlePath) + assert(prefixRemoved) + + return .init( type: .swiftSDK, version: self.bundleVersion, variants: [ .init( - path: FilePath(artifactID).appending(self.targetTriple.triple).string, + path: relativePath.string, supportedTriples: hostTriples.map { $0.map(\.triple) } - ), + ) ] - ), - ] + ) + } ) ) ) @@ -121,7 +141,7 @@ extension SwiftSDKGenerator { } /// Generates an `SDKSettings.json` file that looks like this: - /// + /// /// ```json /// { /// "CanonicalName" : "-swift-linux-[gnu|gnueabihf]", diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index d025768..f4c953c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -51,10 +51,10 @@ extension SwiftSDKGenerator { try self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath) let excludes = - unusedTargetPlatforms.map { "--exclude usr/lib/swift/\($0)" } + - unusedTargetPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } + - unusedHostBinaries.map { "--exclude usr/bin/\($0)" } + - unusedHostLibraries.map { "--exclude usr/lib/\($0)" } + unusedTargetPlatforms.map { "--exclude usr/lib/swift/\($0)" } + + unusedTargetPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } + + unusedHostBinaries.map { "--exclude usr/bin/\($0)" } + + unusedHostLibraries.map { "--exclude usr/lib/\($0)" } if hostSwiftPackagePath.string.contains("tar.gz") { try await Shell.run( @@ -103,12 +103,15 @@ extension SwiftSDKGenerator { try await inTemporaryDirectory { fs, tmpDir in try await fs.unpack(file: targetSwiftPackagePath, into: tmpDir) try await fs.copyTargetSwift( - from: tmpDir.appending(relativePathToRoot).appending("usr"), sdkDirPath: sdkDirPath + from: tmpDir.appending(relativePathToRoot).appending("usr"), + sdkDirPath: sdkDirPath ) } } - func prepareLLDLinker(_ engine: QueryEngine, llvmArtifact: DownloadableArtifacts.Item) async throws { + func prepareLLDLinker(_ engine: QueryEngine, llvmArtifact: DownloadableArtifacts.Item) + async throws + { logger.info("Unpacking and copying `lld` linker...") let pathsConfiguration = self.pathsConfiguration let targetOS = self.targetTriple.os @@ -120,23 +123,27 @@ extension SwiftSDKGenerator { let unpackedLLDPath: FilePath if llvmArtifact.isPrebuilt { - unpackedLLDPath = try await engine[TarExtractQuery( - file: llvmArtifact.localPath, - into: untarDestination, - outputBinarySubpath: ["bin", "lld"], - stripComponents: 1 - )].path + unpackedLLDPath = try await engine[ + TarExtractQuery( + file: llvmArtifact.localPath, + into: untarDestination, + outputBinarySubpath: ["bin", "lld"], + stripComponents: 1 + ) + ].path } else { try await self.untar( file: llvmArtifact.localPath, into: untarDestination, stripComponents: 1 ) - unpackedLLDPath = try await engine[CMakeBuildQuery( - sourcesDirectory: untarDestination, - outputBinarySubpath: ["bin", "lld"], - options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=''" - )].path + unpackedLLDPath = try await engine[ + CMakeBuildQuery( + sourcesDirectory: untarDestination, + outputBinarySubpath: ["bin", "lld"], + options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=''" + ) + ].path } let toolchainLLDPath: FilePath diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index 0db1d5e..b14b61c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -58,19 +58,22 @@ public actor SwiftSDKGenerator { } private let fileManager = FileManager.default - private static let dockerCommand = ProcessInfo.processInfo.environment["SWIFT_SDK_GENERATOR_CONTAINER_RUNTIME"] ?? "docker" + private static let dockerCommand = + ProcessInfo.processInfo.environment["SWIFT_SDK_GENERATOR_CONTAINER_RUNTIME"] ?? "docker" public static func getCurrentTriple(isVerbose: Bool) throws -> Triple { let current = UnixName.current! let cpu = current.machine #if os(macOS) - let darwinVersion = current.release - let darwinTriple = Triple("\(cpu)-apple-darwin\(darwinVersion)") - return Triple("\(cpu)-apple-macos\(darwinTriple._macOSVersion?.description ?? "")") + let darwinVersion = current.release + let darwinTriple = Triple("\(cpu)-apple-darwin\(darwinVersion)") + return Triple("\(cpu)-apple-macos\(darwinTriple._macOSVersion?.description ?? "")") #elseif os(Linux) - return Triple("\(cpu)-unknown-linux-gnu") + return Triple("\(cpu)-unknown-linux-gnu") #else - fatalError("Triple detection not implemented for the platform that this generator was built on.") + fatalError( + "Triple detection not implemented for the platform that this generator was built on." + ) #endif } @@ -132,7 +135,7 @@ public actor SwiftSDKGenerator { func withDockerContainer( fromImage imageName: String, - _ body: @Sendable (String) async throws -> () + _ body: @Sendable (String) async throws -> Void ) async throws { let containerID = try await launchDockerContainer(imageName: imageName) try await withAsyncThrowing { @@ -176,19 +179,25 @@ public actor SwiftSDKGenerator { } func findSymlinks(at directory: FilePath) throws -> [(FilePath, FilePath)] { - guard let enumerator = fileManager.enumerator( - at: URL(fileURLWithPath: directory.string), - includingPropertiesForKeys: [.isSymbolicLinkKey] - ) else { return [] } + guard + let enumerator = fileManager.enumerator( + at: URL(fileURLWithPath: directory.string), + includingPropertiesForKeys: [.isSymbolicLinkKey] + ) + else { return [] } var result = [(FilePath, FilePath)]() for case let url as URL in enumerator { - guard let isSymlink = try url.resourceValues(forKeys: [.isSymbolicLinkKey]) - .isSymbolicLink else { continue } + guard + let isSymlink = try url.resourceValues(forKeys: [.isSymbolicLinkKey]) + .isSymbolicLink + else { continue } if isSymlink { let path = url.path - try result.append((FilePath(path), FilePath(self.fileManager.destinationOfSymbolicLink(atPath: url.path)))) + try result.append( + (FilePath(path), FilePath(self.fileManager.destinationOfSymbolicLink(atPath: url.path))) + ) } } @@ -227,7 +236,10 @@ public actor SwiftSDKGenerator { } func gunzip(file: FilePath, into directoryPath: FilePath) async throws { - try await Shell.run(#"cd "\#(directoryPath)" && gzip -d "\#(file)""#, shouldLogCommands: self.isVerbose) + try await Shell.run( + #"cd "\#(directoryPath)" && gzip -d "\#(file)""#, + shouldLogCommands: self.isVerbose + ) } func untar( @@ -254,7 +266,10 @@ public actor SwiftSDKGenerator { if isVerbose { let cmd = "ls \(tmp)" let lsOutput = try await Shell.readStdout(cmd) - logger.debug("Files unpacked from deb file", metadata: ["cmd": .string(cmd), "output": .string(lsOutput)]) + logger.debug( + "Files unpacked from deb file", + metadata: ["cmd": .string(cmd), "output": .string(lsOutput)] + ) } try await Shell.run( diff --git a/Sources/SwiftSDKGenerator/PathsConfiguration.swift b/Sources/SwiftSDKGenerator/PathsConfiguration.swift index cb898d0..bd0d12b 100644 --- a/Sources/SwiftSDKGenerator/PathsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PathsConfiguration.swift @@ -15,7 +15,8 @@ import struct SystemPackage.FilePath public struct PathsConfiguration: Sendable { init(sourceRoot: FilePath, artifactID: String, targetTriple: Triple) { self.sourceRoot = sourceRoot - self.artifactBundlePath = sourceRoot + self.artifactBundlePath = + sourceRoot .appending("Bundles") .appending("\(artifactID).artifactbundle") self.artifactsCachePath = sourceRoot.appending("Artifacts") diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index f10b818..7f64509 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -11,9 +11,10 @@ //===----------------------------------------------------------------------===// public enum LinuxDistribution: Hashable, Sendable { - public enum Name: String { + public enum Name: String, Sendable { case rhel case ubuntu + case debian } public enum RHEL: String, Sendable { @@ -34,7 +35,10 @@ public enum LinuxDistribution: Hashable, Sendable { case "24.04": self = .noble default: - throw GeneratorError.unknownLinuxDistribution(name: LinuxDistribution.Name.ubuntu.rawValue, version: version) + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.ubuntu.rawValue, + version: version + ) } } @@ -48,7 +52,8 @@ public enum LinuxDistribution: Hashable, Sendable { public var requiredPackages: [String] { switch self { - case .focal: return [ + case .focal: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -61,7 +66,8 @@ public enum LinuxDistribution: Hashable, Sendable { "zlib1g", "zlib1g-dev", ] - case .jammy: return [ + case .jammy: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -74,7 +80,8 @@ public enum LinuxDistribution: Hashable, Sendable { "zlib1g", "zlib1g-dev", ] - case .noble: return [ + case .noble: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -91,8 +98,66 @@ public enum LinuxDistribution: Hashable, Sendable { } } + public enum Debian: String, Sendable { + case bullseye + case bookworm + + init(version: String) throws { + switch version { + case "11": self = .bullseye + case "12": self = .bookworm + default: + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.debian.rawValue, + version: version + ) + } + } + + var version: String { + switch self { + case .bullseye: return "11" + case .bookworm: return "12" + } + } + + public var requiredPackages: [String] { + switch self { + case .bullseye: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-10-dev", + "libicu67", + "libicu-dev", + "libstdc++-10-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + case .bookworm: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-12-dev", + "libicu72", + "libicu-dev", + "libstdc++-12-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + } + } + } + case rhel(RHEL) case ubuntu(Ubuntu) + case debian(Debian) public init(name: Name, version: String) throws { switch name { @@ -104,6 +169,9 @@ public enum LinuxDistribution: Hashable, Sendable { case .ubuntu: self = try .ubuntu(Ubuntu(version: version)) + + case .debian: + self = try .debian(Debian(version: version)) } } @@ -111,6 +179,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case .rhel: return .rhel case .ubuntu: return .ubuntu + case .debian: return .debian } } @@ -118,6 +187,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return rhel.rawValue case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } @@ -125,12 +195,13 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return "rhel-\(rhel.rawValue)" case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } } -public extension LinuxDistribution.Name { - init(nameString: String) throws { +extension LinuxDistribution.Name { + public init(nameString: String) throws { guard let name = LinuxDistribution.Name(rawValue: nameString) else { throw GeneratorError.unknownLinuxDistribution(name: nameString, version: nil) } @@ -144,7 +215,7 @@ extension LinuxDistribution: CustomStringConvertible { switch self { case .rhel: versionComponent = self.release.uppercased() - case .ubuntu: + case .ubuntu, .debian: versionComponent = self.release.capitalized } @@ -157,6 +228,7 @@ extension LinuxDistribution.Name: CustomStringConvertible { switch self { case .rhel: return "RHEL" case .ubuntu: return "Ubuntu" + case .debian: return "Debian" } } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift b/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift index fd7e1c3..55dec55 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift @@ -16,12 +16,12 @@ public typealias Triple = Helpers.Triple extension Triple: @unchecked Sendable {} -public extension Triple { - init(arch: Arch, vendor: Vendor?, os: OS, environment: Environment) { +extension Triple { + public init(arch: Arch, vendor: Vendor?, os: OS, environment: Environment) { self.init("\(arch)-\(vendor?.rawValue ?? "unknown")-\(os)-\(environment)", normalizing: true) } - init(arch: Arch, vendor: Vendor?, os: OS) { + public init(arch: Arch, vendor: Vendor?, os: OS) { self.init("\(arch)-\(vendor?.rawValue ?? "unknown")-\(os)", normalizing: true) } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 85ca59a..3459126 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import Logging + import struct Foundation.URL public struct VersionsConfiguration: Sendable { @@ -18,13 +20,16 @@ public struct VersionsConfiguration: Sendable { swiftBranch: String? = nil, lldVersion: String, linuxDistribution: LinuxDistribution, - targetTriple: Triple + targetTriple: Triple, + logger: Logger ) throws { self.swiftVersion = swiftVersion self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" self.lldVersion = lldVersion self.linuxDistribution = linuxDistribution - self.linuxArchSuffix = targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + self.linuxArchSuffix = + targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + self.logger = logger } let swiftVersion: String @@ -32,18 +37,33 @@ public struct VersionsConfiguration: Sendable { let lldVersion: String let linuxDistribution: LinuxDistribution let linuxArchSuffix: String + private let logger: Logger var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): - return "ubuntu\(ubuntu.version)\(self.linuxArchSuffix)" + return "ubuntu\(ubuntu.version)" + case let .debian(debian): + if debian.version == "11" { + // Ubuntu 20.04 toolchain is binary compatible with Debian 11 + return "ubuntu20.04" + } else if self.swiftVersion.hasPrefix("5.9") || self.swiftVersion == "5.10" { + // Ubuntu 22.04 toolchain is binary compatible with Debian 12 + return "ubuntu22.04" + } + return "debian\(debian.version)" case let .rhel(rhel): - return "\(rhel.rawValue)\(self.linuxArchSuffix)" + return rhel.rawValue } } + var swiftPlatformAndSuffix: String { + return "\(self.swiftPlatform)\(self.linuxArchSuffix)" + } + func swiftDistributionName(platform: String? = nil) -> String { - "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatform)" + return + "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatformAndSuffix)" } func swiftDownloadURL( @@ -51,22 +71,18 @@ public struct VersionsConfiguration: Sendable { platform: String? = nil, fileExtension: String = "tar.gz" ) -> URL { - let computedSubdirectory: String - switch self.linuxDistribution { - case let .ubuntu(ubuntu): - computedSubdirectory = "ubuntu\(ubuntu.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" - case let .rhel(rhel): - computedSubdirectory = rhel.rawValue - } + let computedPlatform = platform ?? self.swiftPlatformAndSuffix + let computedSubdirectory = + subdirectory + ?? computedPlatform.replacingOccurrences(of: ".", with: "") return URL( string: """ - https://download.swift.org/\( - self.swiftBranch - )/\( - subdirectory ?? computedSubdirectory - )/swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: platform)).\(fileExtension) - """ + https://download.swift.org/\( + self.swiftBranch + )/\(computedSubdirectory)/\ + swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: computedPlatform)).\(fileExtension) + """ )! } diff --git a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift index a5d5e20..d0d8652 100644 --- a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import Helpers + import struct SystemPackage.FilePath struct CMakeBuildQuery: CachingQuery { diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift index 041737c..adc8eb5 100644 --- a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift @@ -10,9 +10,10 @@ // //===----------------------------------------------------------------------===// -import class Foundation.ByteCountFormatter -import Logging import Helpers +import Logging + +import class Foundation.ByteCountFormatter import struct SystemPackage.FilePath struct DownloadArtifactQuery: Query { @@ -22,9 +23,13 @@ struct DownloadArtifactQuery: Query { let logger: Logger func run(engine: QueryEngine) async throws -> FilePath { - logger.info("Downloading remote artifact not available in local cache", metadata: ["remoteUrl": .string(self.artifact.remoteURL.absoluteString)]) + logger.info( + "Downloading remote artifact not available in local cache", + metadata: ["remoteUrl": .string(self.artifact.remoteURL.absoluteString)] + ) let stream = self.httpClient.streamDownloadProgress( - from: self.artifact.remoteURL, to: self.artifact.localPath + from: self.artifact.remoteURL, + to: self.artifact.localPath ) .removeDuplicates(by: didProgressChangeSignificantly) ._throttle(for: .seconds(1)) @@ -39,15 +44,17 @@ struct DownloadArtifactQuery: Query { let byteCountFormatter = ByteCountFormatter() if let total = progress.totalBytes { - logger.debug(""" - \(artifact.remoteURL.lastPathComponent) \( - byteCountFormatter - .string(fromByteCount: Int64(progress.receivedBytes)) - )/\( - byteCountFormatter - .string(fromByteCount: Int64(total)) + logger.debug( + """ + \(artifact.remoteURL.lastPathComponent) \( + byteCountFormatter + .string(fromByteCount: Int64(progress.receivedBytes)) + )/\( + byteCountFormatter + .string(fromByteCount: Int64(total)) + ) + """ ) - """) } else { logger.debug( "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))" diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift index 41d9a1a..43e3d4e 100644 --- a/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift @@ -10,8 +10,9 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.URL import Helpers + +import struct Foundation.URL import struct SystemPackage.FilePath struct DownloadFileQuery: Query { diff --git a/Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift b/Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift index cf5828d..4eeabf5 100644 --- a/Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2022-202Apple Inc. and the Swift project authors +// Copyright (c) 2022-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information diff --git a/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift b/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift index b693711..ad855ea 100644 --- a/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift +++ b/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift @@ -73,8 +73,8 @@ struct DestinationV3: Encodable { } /// Represents v4 schema of `swift-sdk.json` (previously `destination.json`) files used for cross-compilation. -public struct SwiftSDKMetadataV4: Encodable { - public struct TripleProperties: Encodable { +package struct SwiftSDKMetadataV4: Encodable { + package struct TripleProperties: Encodable { /// Path relative to `swift-sdk.json` containing SDK root. var sdkRootPath: String @@ -98,5 +98,5 @@ public struct SwiftSDKMetadataV4: Encodable { let schemaVersion = "4.0" /// Mapping of triple strings to corresponding properties of such target triple. - let targetTriples: [String: TripleProperties] + var targetTriples: [String: TripleProperties] } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 9af0afc..af3ecc4 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -11,18 +11,19 @@ //===----------------------------------------------------------------------===// import Foundation -import Logging import Helpers +import Logging + import struct SystemPackage.FilePath -public struct LinuxRecipe: SwiftSDKRecipe { - public enum TargetSwiftSource: Sendable { +package struct LinuxRecipe: SwiftSDKRecipe { + package enum TargetSwiftSource: Sendable { case docker(baseSwiftDockerImage: String) case localPackage(FilePath) case remoteTarball } - public enum HostSwiftSource: Sendable, Equatable { + package enum HostSwiftSource: Sendable, Equatable { case localPackage(FilePath) case remoteTarball case preinstalled @@ -34,7 +35,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { let targetSwiftSource: TargetSwiftSource let hostSwiftSource: HostSwiftSource let versionsConfiguration: VersionsConfiguration - public let logger: Logger + package let logger: Logger var shouldUseDocker: Bool { if case .docker = self.targetSwiftSource { @@ -43,7 +44,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { return false } - public init( + package init( targetTriple: Triple, hostTriple: Triple, linuxDistribution: LinuxDistribution, @@ -62,7 +63,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { swiftBranch: swiftBranch, lldVersion: lldVersion, linuxDistribution: linuxDistribution, - targetTriple: targetTriple + targetTriple: targetTriple, + logger: logger ) let targetSwiftSource: LinuxRecipe.TargetSwiftSource @@ -96,7 +98,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) } - public init( + package init( mainTargetTriple: Triple, mainHostTriple: Triple, linuxDistribution: LinuxDistribution, @@ -114,7 +116,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { self.logger = logger } - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) { + package func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple, isForEmbeddedSwift: Bool) { if self.hostSwiftSource == .preinstalled { toolset.rootPath = nil } @@ -128,6 +130,11 @@ public struct LinuxRecipe: SwiftSDKRecipe { } else { swiftCompilerOptions.append("-use-ld=lld") + // 32-bit architectures require libatomic + if let arch = targetTriple.arch, arch.is32Bit { + swiftCompilerOptions.append("-latomic") + } + if self.hostSwiftSource != .preinstalled { toolset.linker = Toolset.ToolProperties(path: "ld.lld") } @@ -139,20 +146,22 @@ public struct LinuxRecipe: SwiftSDKRecipe { toolset.librarian = Toolset.ToolProperties(path: "llvm-ar") } - public func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + package func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) { var relativeSDKDir = self.sdkDirPath(paths: paths) guard relativeSDKDir.removePrefix(paths.swiftSDKRootPath) else { fatalError("The SDK directory path must be a subdirectory of the Swift SDK root path.") } - metadata.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string - metadata.swiftStaticResourcesPath = relativeSDKDir.appending("usr/lib/swift_static").string + metadata.targetTriples[targetTriple.triple]?.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string + metadata.targetTriples[targetTriple.triple]?.swiftStaticResourcesPath = + relativeSDKDir.appending("usr/lib/swift_static").string } - public var defaultArtifactID: String { + package var defaultArtifactID: String { """ \(self.versionsConfiguration.swiftVersion)_\(self.linuxDistribution.name.rawValue)_\( self.linuxDistribution @@ -172,7 +181,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { var items: [DownloadableArtifacts.Item] = [] if self.hostSwiftSource != .preinstalled && self.mainHostTriple.os != .linux - && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") { + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.") + { items.append(artifacts.hostLLVM) } @@ -211,17 +221,18 @@ public struct LinuxRecipe: SwiftSDKRecipe { return [self.mainHostTriple] } - public func makeSwiftSDK( + package func makeSwiftSDK( generator: SwiftSDKGenerator, engine: QueryEngine, httpClient client: some HTTPClientProtocol ) async throws -> SwiftSDKProduct { if self.linuxDistribution.name == .rhel && self.mainTargetTriple.archName == "armv7" { throw GeneratorError.distributionDoesNotSupportArchitecture( - self.linuxDistribution, targetArchName: self.mainTargetTriple.archName + self.linuxDistribution, + targetArchName: self.mainTargetTriple.archName ) } - + let sdkDirPath = self.sdkDirPath(paths: generator.pathsConfiguration) if !generator.isIncremental { try await generator.removeRecursively(at: sdkDirPath) @@ -243,24 +254,35 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) if !self.shouldUseDocker { - guard case let .ubuntu(version) = linuxDistribution else { - throw GeneratorError + switch linuxDistribution { + case .ubuntu(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + case .debian(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + default: + throw + GeneratorError .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) } - - try await generator.downloadUbuntuPackages( - client, - engine, - requiredPackages: version.requiredPackages, - versionsConfiguration: self.versionsConfiguration, - sdkDirPath: sdkDirPath - ) } switch self.hostSwiftSource { case let .localPackage(filePath): try await generator.rsync( - from: filePath.appending("usr"), to: generator.pathsConfiguration.toolchainDirPath + from: filePath.appending("usr"), + to: generator.pathsConfiguration.toolchainDirPath ) case .remoteTarball: try await generator.unpackHostSwift( @@ -279,26 +301,43 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) case let .localPackage(filePath): try await generator.copyTargetSwift( - from: filePath.appending("usr"), sdkDirPath: sdkDirPath + from: filePath.appending("usr"), + sdkDirPath: sdkDirPath ) case .remoteTarball: try await generator.unpackTargetSwiftPackage( targetSwiftPackagePath: downloadableArtifacts.targetSwift.localPath, - relativePathToRoot: [FilePath.Component(self.versionsConfiguration.swiftDistributionName())!], + relativePathToRoot: [ + FilePath.Component(self.versionsConfiguration.swiftDistributionName())! + ], sdkDirPath: sdkDirPath ) } + logger.info("Removing unused toolchain components from target SDK...") + try await generator.removeToolchainComponents( + sdkDirPath, + platforms: unusedTargetPlatforms, + libraries: unusedHostLibraries, + binaries: unusedHostBinaries + ) + + try await generator.createLibSymlink(sdkDirPath: sdkDirPath) try await generator.fixAbsoluteSymlinks(sdkDirPath: sdkDirPath) // Swift 6.1 and later do not throw warnings about the SDKSettings.json file missing, // so they don't need this file. if self.versionsConfiguration.swiftVersion.hasAnyPrefix(from: ["5.9", "5.10", "6.0"]) { - try await generator.generateSDKSettingsFile(sdkDirPath: sdkDirPath, distribution: linuxDistribution) + try await generator.generateSDKSettingsFile( + sdkDirPath: sdkDirPath, + distribution: linuxDistribution + ) } if self.hostSwiftSource != .preinstalled { - if self.mainHostTriple.os != .linux && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") { + if self.mainHostTriple.os != .linux + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.") + { try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM) } @@ -306,7 +345,9 @@ public struct LinuxRecipe: SwiftSDKRecipe { try await generator.symlinkClangHeaders() } - let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending("swift-autolink-extract") + let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending( + "swift-autolink-extract" + ) if await !generator.doesFileExist(at: autolinkExtractPath) { logger.info("Fixing `swift-autolink-extract` symlink...") diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 17908a4..6b11451 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -10,26 +10,30 @@ // //===----------------------------------------------------------------------===// -import Logging import Helpers +import Logging + import struct SystemPackage.FilePath -public struct SwiftSDKProduct { +package struct SwiftSDKProduct { let sdkDirPath: FilePath /// Array of supported host triples. `nil` indicates the SDK can be universally used. let hostTriples: [Triple]? } /// A protocol describing a set of platform specific instructions to make a Swift SDK -public protocol SwiftSDKRecipe: Sendable { +package protocol SwiftSDKRecipe: Sendable { /// Update the given toolset with platform specific options func applyPlatformOptions( - toolset: inout Toolset, targetTriple: Triple + toolset: inout Toolset, + targetTriple: Triple, + isForEmbeddedSwift: Bool ) func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) /// The default identifier of the Swift SDK @@ -39,15 +43,27 @@ public protocol SwiftSDKRecipe: Sendable { var logger: Logger { get } /// The main entrypoint of the recipe to make a Swift SDK - func makeSwiftSDK(generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol) async throws - -> SwiftSDKProduct + func makeSwiftSDK( + generator: SwiftSDKGenerator, + engine: QueryEngine, + httpClient: some HTTPClientProtocol + ) async throws -> SwiftSDKProduct + + var shouldSupportEmbeddedSwift: Bool { get } } -public extension SwiftSDKRecipe { - func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) {} - func applyPlatformOptions( +extension SwiftSDKRecipe { + package func applyPlatformOptions( + toolset: inout Toolset, + targetTriple: Triple, + isForEmbeddedSwift: Bool + ) {} + package func applyPlatformOptions( metadata: inout SwiftSDKMetadataV4.TripleProperties, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) {} + + package var shouldSupportEmbeddedSwift: Bool { false } } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index 21b2485..8497f9b 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -10,28 +10,29 @@ // //===----------------------------------------------------------------------===// -import Logging import Helpers +import Logging + import struct SystemPackage.FilePath -public struct WebAssemblyRecipe: SwiftSDKRecipe { +package struct WebAssemblyRecipe: SwiftSDKRecipe { let hostSwiftPackage: HostToolchainPackage? let targetSwiftPackagePath: FilePath let wasiSysroot: FilePath let swiftVersion: String - public let logger: Logger + package let logger: Logger - public struct HostToolchainPackage: Sendable { + package struct HostToolchainPackage: Sendable { let path: FilePath let triple: Triple - public init(path: FilePath, triple: Triple) { + package init(path: FilePath, triple: Triple) { self.path = path self.triple = triple } } - public init( + package init( hostSwiftPackage: HostToolchainPackage?, targetSwiftPackagePath: FilePath, wasiSysroot: FilePath, @@ -45,13 +46,35 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { self.logger = logger } - public var defaultArtifactID: String { + package var defaultArtifactID: String { "\(self.swiftVersion)_wasm" } - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) { + package let shouldSupportEmbeddedSwift = true + + package func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple, isForEmbeddedSwift: Bool) { // We only support static linking for WebAssembly for now, so make it the default. toolset.swiftCompiler = Toolset.ToolProperties(extraCLIOptions: ["-static-stdlib"]) + + if isForEmbeddedSwift { + let ccOptions = ["-D__EMBEDDED_SWIFT__"] + toolset.cCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) + toolset.cxxCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) + + toolset.swiftCompiler?.extraCLIOptions?.append( + contentsOf: [ + "-enable-experimental-feature", "Embedded", "-wmo", + ] + ) + + toolset.swiftCompiler?.extraCLIOptions?.append( + // libraries required for concurrency + contentsOf: ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { + ["-Xlinker", $0] + } + ) + } + if targetTriple.environmentName == "threads" { // Enable features required for threading support let ccOptions = [ @@ -59,9 +82,11 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { "-pthread", "-ftls-model=local-exec", ] // Tell LLVM codegen in swiftc to enable those features via clang options - toolset.swiftCompiler?.extraCLIOptions?.append(contentsOf: ccOptions.flatMap { - ["-Xcc", $0] - }) + toolset.swiftCompiler?.extraCLIOptions?.append( + contentsOf: ccOptions.flatMap { + ["-Xcc", $0] + } + ) // Tell the C and C++ compilers to enable those features toolset.cCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) toolset.cxxCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) @@ -79,20 +104,37 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } } - public func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + package func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) { var relativeToolchainDir = paths.toolchainDirPath guard relativeToolchainDir.removePrefix(paths.swiftSDKRootPath) else { - fatalError("The toolchain bin directory path must be a subdirectory of the Swift SDK root path.") + fatalError( + "The toolchain bin directory path must be a subdirectory of the Swift SDK root path." + ) } - metadata.swiftStaticResourcesPath = relativeToolchainDir.appending("usr/lib/swift_static").string - metadata.swiftResourcesPath = metadata.swiftStaticResourcesPath + + var tripleProperties = metadata.targetTriples[targetTriple.triple]! + tripleProperties.swiftStaticResourcesPath = + relativeToolchainDir.appending("usr/lib/swift_static").string + tripleProperties.swiftResourcesPath = + isForEmbeddedSwift + ? relativeToolchainDir.appending("usr/lib/swift").string + : tripleProperties.swiftStaticResourcesPath + + var finalTriple = targetTriple + if isForEmbeddedSwift { + metadata.targetTriples.removeValue(forKey: targetTriple.triple) + finalTriple = Triple("wasm32-unknown-wasip1") + } + + metadata.targetTriples[finalTriple.triple] = tripleProperties } - public func makeSwiftSDK( + package func makeSwiftSDK( generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol @@ -104,7 +146,10 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { var hostTriples: [Triple]? = nil if let hostSwiftPackage { hostTriples = [hostSwiftPackage.triple] - try await generator.rsync(from: hostSwiftPackage.path.appending("usr"), to: pathsConfiguration.toolchainDirPath) + try await generator.rsync( + from: hostSwiftPackage.path.appending("usr"), + to: pathsConfiguration.toolchainDirPath + ) logger.info("Removing unused toolchain components...") let liblldbNames: [String] = try await { @@ -127,20 +172,36 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { try await self.mergeTargetSwift(from: targetSwiftLibPath, generator: generator) } else { // Simply copy the target Swift package into the SDK bundle when building host-agnostic SDK. - try await generator.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath.appending("usr")) - try await generator.copy(from: targetSwiftLibPath, to: pathsConfiguration.toolchainDirPath.appending("usr/lib")) + try await generator.createDirectoryIfNeeded( + at: pathsConfiguration.toolchainDirPath.appending("usr") + ) + try await generator.copy( + from: targetSwiftLibPath, + to: pathsConfiguration.toolchainDirPath.appending("usr/lib") + ) } - let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending("swift-autolink-extract") + let autolinkExtractPath = pathsConfiguration.toolchainBinDirPath.appending( + "swift-autolink-extract" + ) // WebAssembly object file requires `swift-autolink-extract` if await !generator.doesFileExist(at: autolinkExtractPath), - await generator.doesFileExist(at: generator.pathsConfiguration.toolchainBinDirPath.appending("swift")) + await generator.doesFileExist( + at: generator.pathsConfiguration.toolchainBinDirPath.appending("swift") + ) { logger.info("Fixing `swift-autolink-extract` symlink...") try await generator.createSymlink(at: autolinkExtractPath, pointingTo: "swift") } + // Embedded Swift looks up clang compiler-rt in a different path. + let embeddedCompilerRTPath = pathsConfiguration.toolchainDirPath.appending( + "usr/lib/swift/clang/lib/wasip1" + ) + + try await generator.createSymlink(at: embeddedCompilerRTPath, pointingTo: "wasi") + // Copy the WASI sysroot into the SDK bundle. let sdkDirPath = pathsConfiguration.swiftSDKRootPath.appending("WASI.sdk") try await generator.rsyncContents(from: self.wasiSysroot, to: sdkDirPath) @@ -156,17 +217,34 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { ("clang", pathsConfiguration.toolchainDirPath.appending("usr/lib"), false), ("swift/clang", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift"), false), ("swift/wasi", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift"), false), - ("swift_static/clang", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), false), - ("swift_static/wasi", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), false), - ("swift_static/shims", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), false), + ( + "swift_static/clang", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), + false + ), + ( + "swift_static/wasi", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), + false + ), + ( + "swift_static/shims", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), + false + ), // Mark CoreFoundation as optional until we set up build system to build it for WebAssembly - ("swift_static/CoreFoundation", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), true), + ( + "swift_static/CoreFoundation", + pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), true + ), ] { - if isOptional, await !(generator.doesFileExist(at: distributionPath.appending(pathWithinPackage))) { + if isOptional, + await !(generator.doesFileExist(at: distributionPath.appending(pathWithinPackage))) + { logger.debug("Skipping optional path \(pathWithinPackage)") continue } - try await generator.rsync(from: distributionPath.appending(pathWithinPackage), to: pathWithinSwiftSDK) + try await generator.rsync( + from: distributionPath.appending(pathWithinPackage), + to: pathWithinSwiftSDK + ) } } } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index f16b82a..f8f1414 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -12,16 +12,19 @@ import AsyncProcess import Foundation +import Logging import NIOCore -public extension ByteBuffer { - func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { +extension ByteBuffer { + public func unzip(zipPath: String, logger: Logger) async throws -> ByteBuffer? { let result = try await ProcessExecutor.runCollectingOutput( - executable: zipPath, ["-cd"], + executable: zipPath, + ["-cd"], standardInput: [self].async, collectStandardOutput: true, collectStandardError: false, - perStreamCollectionLimitBytes: 20 * 1024 * 1024 + perStreamCollectionLimitBytes: 100 * 1024 * 1024, + logger: logger ) try result.exitReason.throwIfNonZero() diff --git a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift index 2ecae65..6fcdb7b 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift @@ -25,8 +25,9 @@ enum GeneratorError: Error { case distributionDoesNotSupportArchitecture(LinuxDistribution, targetArchName: String) case fileDoesNotExist(FilePath) case fileDownloadFailed(URL, String) - case ubuntuPackagesDecompressionFailure - case ubuntuPackagesParsingFailure(expectedPackages: Int, actual: Int) + case debianPackagesListDownloadRequiresXz + case packagesListDecompressionFailure + case packagesListParsingFailure(expectedPackages: Int, actual: Int) } extension GeneratorError: CustomStringConvertible { @@ -39,7 +40,8 @@ extension GeneratorError: CustomStringConvertible { case let .nonZeroExitCode(exitCode, commandInfo): return "Process launched with \(commandInfo) failed with exit code \(exitCode)" case let .unknownLinuxDistribution(name, version): - return "Linux distribution `\(name)`\(version.map { " with version \($0)" } ?? "")` is not supported by this generator." + return + "Linux distribution `\(name)`\(version.map { " with version \($0)" } ?? "")` is not supported by this generator." case let .unknownMacOSVersion(version): return "macOS version `\(version)` is not supported by this generator." case let .unknownCPUArchitecture(cpu): @@ -48,21 +50,24 @@ extension GeneratorError: CustomStringConvertible { return "LLD version `\(version)` is not supported by this generator." case let .distributionSupportsOnlyDockerGenerator(linuxDistribution): return """ - Target Linux distribution \(linuxDistribution) supports Swift SDK generation only when `--with-docker` flag is \ - passed. - """ + Target Linux distribution \(linuxDistribution) supports Swift SDK generation only when `--with-docker` flag is \ + passed. + """ case let .distributionDoesNotSupportArchitecture(linuxDistribution, targetArchName): return """ - Target Linux distribution \(linuxDistribution) does not support the target architecture: \(targetArchName) - """ + Target Linux distribution \(linuxDistribution) does not support the target architecture: \(targetArchName) + """ case let .fileDoesNotExist(filePath): return "Expected to find a file at path `\(filePath)`." case let .fileDownloadFailed(url, status): - return "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." - case .ubuntuPackagesDecompressionFailure: - return "Failed to decompress the list of Ubuntu packages" - case let .ubuntuPackagesParsingFailure(expected, actual): - return "Failed to parse Ubuntu packages manifest, expected \(expected), found \(actual) packages." + return + "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." + case .debianPackagesListDownloadRequiresXz: + return "Downloading the Debian packages list requires xz, and it is not installed." + case .packagesListDecompressionFailure: + return "Failed to decompress the list of packages." + case let .packagesListParsingFailure(expected, actual): + return "Failed to parse packages manifest, expected \(expected), found \(actual) packages." } } } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift b/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift index a5abfeb..a693d5d 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift @@ -26,7 +26,8 @@ public protocol HTTPClientProtocol: Sendable { /// NOTE: The client will be shutdown after the operation completes, so it /// should not be stored or used outside of the operation. static func with( - http1Only: Bool, _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + http1Only: Bool, + _ body: @Sendable (any HTTPClientProtocol) async throws -> Result ) async throws -> Result /// Download a file from the given URL to the given path. @@ -53,7 +54,9 @@ public protocol HTTPClientProtocol: Sendable { } extension HTTPClientProtocol { - static func with(_ body: @Sendable (any HTTPClientProtocol) async throws -> Result) async throws + static func with( + _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + ) async throws -> Result { try await self.with(http1Only: false, body) @@ -63,112 +66,123 @@ extension HTTPClientProtocol { extension FilePath: @unchecked Sendable {} #if canImport(AsyncHTTPClient) -import AsyncHTTPClient + import AsyncHTTPClient -extension FileDownloadDelegate.Progress: @unchecked Sendable {} + extension FileDownloadDelegate.Progress: @unchecked Sendable {} -extension HTTPClient: HTTPClientProtocol { - public static func with( - http1Only: Bool, _ body: @Sendable (any HTTPClientProtocol) async throws -> Result - ) async throws -> Result { - var configuration = HTTPClient.Configuration(redirectConfiguration: .follow(max: 5, allowCycles: false)) - if http1Only { - configuration.httpVersion = .http1Only - } - let client = HTTPClient(eventLoopGroupProvider: .singleton, configuration: configuration) - return try await withAsyncThrowing { - try await body(client) - } defer: { - try await client.shutdown() + extension HTTPClient: HTTPClientProtocol { + public static func with( + http1Only: Bool, + _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + ) async throws -> Result { + var configuration = HTTPClient.Configuration( + redirectConfiguration: .follow(max: 5, allowCycles: false) + ) + if http1Only { + configuration.httpVersion = .http1Only + } + let client = HTTPClient(eventLoopGroupProvider: .singleton, configuration: configuration) + return try await withAsyncThrowing { + try await body(client) + } defer: { + try await client.shutdown() + } } - } - public func get(url: String) async throws -> (status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer?) { - let response = try await self.get(url: url).get() - return (status: response.status, body: response.body) - } + public func get(url: String) async throws -> ( + status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer? + ) { + let response = try await self.get(url: url).get() + return (status: response.status, body: response.body) + } - public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { - var headRequest = HTTPClientRequest(url: url) - headRequest.method = .HEAD - headRequest.headers = ["Accept": "*/*", "User-Agent": "Swift SDK Generator"] - return try await self.execute(headRequest, deadline: .distantFuture).status == .ok - } + public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { + var headRequest = HTTPClientRequest(url: url) + headRequest.method = .HEAD + headRequest.headers = ["Accept": "*/*", "User-Agent": "Swift SDK Generator"] + return try await self.execute(headRequest, deadline: .distantFuture).status == .ok + } - public func downloadFile( - from url: URL, - to path: FilePath - ) async throws { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<(), Error>) in - do { - let delegate = try FileDownloadDelegate( - path: path.string, - reportHead: { task, responseHead in - if responseHead.status != .ok { - task.fail(reason: GeneratorError.fileDownloadFailed(url, responseHead.status.description)) + public func downloadFile( + from url: URL, + to path: FilePath + ) async throws { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<(), Error>) in + do { + let delegate = try FileDownloadDelegate( + path: path.string, + reportHead: { task, responseHead in + if responseHead.status != .ok { + task.fail( + reason: GeneratorError.fileDownloadFailed(url, responseHead.status.description) + ) + } + } + ) + let request = try HTTPClient.Request(url: url) + + execute(request: request, delegate: delegate).futureResult.whenComplete { + switch $0 { + case let .failure(error): + continuation.resume(throwing: error) + case .success: + continuation.resume(returning: ()) } } - ) - let request = try HTTPClient.Request(url: url) - - execute(request: request, delegate: delegate).futureResult.whenComplete { - switch $0 { - case let .failure(error): - continuation.resume(throwing: error) - case .success: - continuation.resume(returning: ()) - } + } catch { + continuation.resume(throwing: error) } - } catch { - continuation.resume(throwing: error) } } - } - public func streamDownloadProgress( - from url: URL, - to path: FilePath - ) -> AsyncThrowingStream { - .init { continuation in - do { - let delegate = try FileDownloadDelegate( - path: path.string, - reportHead: { - if $0.status != .ok { - continuation - .finish(throwing: FileOperationError.downloadFailed(url, $0.status.description)) + public func streamDownloadProgress( + from url: URL, + to path: FilePath + ) -> AsyncThrowingStream { + .init { continuation in + do { + let delegate = try FileDownloadDelegate( + path: path.string, + reportHead: { + if $0.status != .ok { + continuation + .finish(throwing: FileOperationError.downloadFailed(url, $0.status.description)) + } + }, + reportProgress: { + continuation.yield( + DownloadProgress(totalBytes: $0.totalBytes, receivedBytes: $0.receivedBytes) + ) + } + ) + let request = try HTTPClient.Request(url: url) + + execute(request: request, delegate: delegate).futureResult.whenComplete { + switch $0 { + case let .failure(error): + continuation.finish(throwing: error) + case let .success(finalProgress): + continuation.yield( + DownloadProgress( + totalBytes: finalProgress.totalBytes, + receivedBytes: finalProgress.receivedBytes + ) + ) + continuation.finish() } - }, - reportProgress: { - continuation.yield( - DownloadProgress(totalBytes: $0.totalBytes, receivedBytes: $0.receivedBytes) - ) - } - ) - let request = try HTTPClient.Request(url: url) - - execute(request: request, delegate: delegate).futureResult.whenComplete { - switch $0 { - case let .failure(error): - continuation.finish(throwing: error) - case let .success(finalProgress): - continuation.yield( - DownloadProgress(totalBytes: finalProgress.totalBytes, receivedBytes: finalProgress.receivedBytes) - ) - continuation.finish() } + } catch { + continuation.finish(throwing: error) } - } catch { - continuation.finish(throwing: error) } } } -} #endif struct OfflineHTTPClient: HTTPClientProtocol { static func with( - http1Only: Bool, _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + http1Only: Bool, + _ body: @Sendable (any HTTPClientProtocol) async throws -> Result ) async throws -> Result { let client = OfflineHTTPClient() return try await body(client) @@ -189,11 +203,19 @@ struct OfflineHTTPClient: HTTPClientProtocol { } } - public func get(url: String) async throws -> (status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer?) { - throw FileOperationError.downloadFailed(URL(string: url)!, "Cannot fetch file with offline client") + public func get(url: String) async throws -> ( + status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer? + ) { + throw FileOperationError.downloadFailed( + URL(string: url)!, + "Cannot fetch file with offline client" + ) } public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { - throw FileOperationError.downloadFailed(URL(string: url)!, "Cannot fetch file with offline client") + throw FileOperationError.downloadFailed( + URL(string: url)!, + "Cannot fetch file with offline client" + ) } } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/Shell.swift b/Sources/SwiftSDKGenerator/SystemUtils/Shell.swift index de62dfb..62bc1ca 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/Shell.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/Shell.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import AsyncProcess + import class Foundation.ProcessInfo import struct SystemPackage.FilePath @@ -125,7 +126,9 @@ struct Shell { try result.exitReason.throwIfNonZero() - guard let stdOutBuffer = result.standardOutput else { throw GeneratorError.noProcessOutput(command) } + guard let stdOutBuffer = result.standardOutput else { + throw GeneratorError.noProcessOutput(command) + } return String(buffer: stdOutBuffer) } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift b/Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift index 3d9135c..9389e6f 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift @@ -1,7 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024-2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + #if canImport(Darwin) -import Darwin + import Darwin #elseif canImport(Glibc) -import Glibc + import Glibc #endif /// libc's `uname` wrapper diff --git a/Sources/SwiftSDKGenerator/SystemUtils/which.swift b/Sources/SwiftSDKGenerator/SystemUtils/which.swift index 472c32b..c856b46 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/which.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/which.swift @@ -14,13 +14,16 @@ import AsyncProcess import Foundation /// Look for an executable using the `which` utility. -/// +/// /// - Parameter executableName: The name of the executable to search for. /// - Throws: Any errors thrown by the ProcessExecutor. /// - Returns: The path to the executable if found, otherwise nil. func which(_ executableName: String) async throws -> String? { let result = try await ProcessExecutor.runCollectingOutput( - executable: "/usr/bin/which", [executableName], collectStandardOutput: true, collectStandardError: false, + executable: "/usr/bin/which", + [executableName], + collectStandardOutput: true, + collectStandardError: false, environment: ProcessInfo.processInfo.environment ) diff --git a/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift b/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift index 1ce58a5..d2d693e 100644 --- a/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift +++ b/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift @@ -43,7 +43,9 @@ final class AsyncByteBufferLineSequenceTests: XCTestCase { func testManyChunksNoNewlineNotDeliveringLastChunk() async throws { for n in 0..<100 { let inputs: [ByteBuffer] = [ByteBuffer(repeating: 0, count: n)] - let lines = try await Array(inputs.async.splitIntoLines(dropLastChunkIfNoNewline: true).strings) + let lines = try await Array( + inputs.async.splitIntoLines(dropLastChunkIfNoNewline: true).strings + ) XCTAssertEqual([], lines) } } @@ -61,11 +63,11 @@ final class AsyncByteBufferLineSequenceTests: XCTestCase { } func testOverlyLongLineIsSplitByDefault() async throws { - var inputs = [ByteBuffer(repeating: UInt8(0), count: 1024 * 1024 - 2)] // almost at the limit + var inputs = [ByteBuffer(repeating: UInt8(0), count: 1024 * 1024 - 2)] // almost at the limit inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) - inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // hitting the limit - inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // over the limit - inputs.append(ByteBuffer(integer: UInt8(ascii: "\n"))) // too late + inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // hitting the limit + inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // over the limit + inputs.append(ByteBuffer(integer: UInt8(ascii: "\n"))) // too late let lines = try await Array( inputs.async.splitIntoLines( dropTerminator: false, diff --git a/Tests/AsyncProcessTests/IntegrationTests.swift b/Tests/AsyncProcessTests/IntegrationTests.swift index 7b5a4ee..922e37a 100644 --- a/Tests/AsyncProcessTests/IntegrationTests.swift +++ b/Tests/AsyncProcessTests/IntegrationTests.swift @@ -19,9 +19,9 @@ import NIOConcurrencyHelpers import XCTest #if canImport(Darwin) -import Darwin + import Darwin #else -import Glibc + import Glibc #endif final class IntegrationTests: XCTestCase { @@ -31,7 +31,8 @@ final class IntegrationTests: XCTestCase { func testTheBasicsWork() async throws { let exe = ProcessExecutor( group: self.group, - executable: "/bin/sh", ["-c", "exit 0"], + executable: "/bin/sh", + ["-c", "exit 0"], standardInput: EOFSequence(), logger: self.logger ) @@ -50,7 +51,8 @@ final class IntegrationTests: XCTestCase { for exitCode in UInt8.min...UInt8.max { let exe = ProcessExecutor( group: self.group, - executable: "/bin/sh", ["-c", "exit \(exitCode)"], + executable: "/bin/sh", + ["-c", "exit \(exitCode)"], standardInput: EOFSequence(), logger: self.logger ) @@ -69,15 +71,16 @@ final class IntegrationTests: XCTestCase { func testSignalsWork() async throws { #if os(Linux) - // workaround for https://github.com/apple/swift-corelibs-foundation/issues/4772 - let signalsToTest: [CInt] = [SIGKILL] + // workaround for https://github.com/apple/swift-corelibs-foundation/issues/4772 + let signalsToTest: [CInt] = [SIGKILL] #else - let signalsToTest: [CInt] = [SIGKILL, SIGTERM, SIGINT] + let signalsToTest: [CInt] = [SIGKILL, SIGTERM, SIGINT] #endif for signal in signalsToTest { let exe = ProcessExecutor( group: self.group, - executable: "/bin/sh", ["-c", "kill -\(signal) $$"], + executable: "/bin/sh", + ["-c", "kill -\(signal) $$"], standardInput: EOFSequence(), logger: self.logger ) @@ -99,7 +102,8 @@ final class IntegrationTests: XCTestCase { let input = AsyncStream.justMakeIt(elementType: ByteBuffer.self) let exe = ProcessExecutor( group: self.group, - executable: "/bin/cat", ["-nu"], // sh", ["-c", "while read -r line; do echo $line; done"], + executable: "/bin/cat", + ["-nu"], // sh", ["-c", "while read -r line; do echo $line; done"], standardInput: input.consumer, logger: self.logger ) @@ -209,7 +213,8 @@ final class IntegrationTests: XCTestCase { let input = AsyncStream.justMakeIt(elementType: ByteBuffer.self) let exe = ProcessExecutor( group: self.group, - executable: "/bin/sh", [], + executable: "/bin/sh", + [], standardInput: input.consumer, logger: self.logger ) @@ -521,7 +526,7 @@ final class IntegrationTests: XCTestCase { func testLogOutputToMetadata() async throws { let sharedRecorder = LogRecorderHandler() var recordedLogger = Logger(label: "recorder", factory: { _ in sharedRecorder }) - recordedLogger.logLevel = .info // don't give us the normal messages + recordedLogger.logLevel = .info // don't give us the normal messages recordedLogger[metadataKey: "yo"] = "hey" try await ProcessExecutor.runLogOutput( @@ -530,20 +535,24 @@ final class IntegrationTests: XCTestCase { ["-c", "echo 1; echo >&2 2; echo 3; echo >&2 4; echo 5; echo >&2 6; echo 7; echo >&2 8;"], standardInput: EOFSequence(), logger: recordedLogger, - logConfiguration: OutputLoggingSettings(logLevel: .critical, to: .metadata(logMessage: "msg", key: "key")) + logConfiguration: OutputLoggingSettings( + logLevel: .critical, + to: .metadata(logMessage: "msg", key: "key") + ) ).throwIfNonZero() XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.level == .critical }) XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.message == "msg" }) XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.metadata["key"] != nil }) XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.metadata["yo"] == "hey" }) - let loggedLines = sharedRecorder.recordedMessages.compactMap { $0.metadata["key"]?.description }.sorted() + let loggedLines = sharedRecorder.recordedMessages.compactMap { $0.metadata["key"]?.description } + .sorted() XCTAssertEqual(["1", "2", "3", "4", "5", "6", "7", "8"], loggedLines) } func testLogOutputToMessage() async throws { let sharedRecorder = LogRecorderHandler() var recordedLogger = Logger(label: "recorder", factory: { _ in sharedRecorder }) - recordedLogger.logLevel = .info // don't give us the normal messages + recordedLogger.logLevel = .info // don't give us the normal messages recordedLogger[metadataKey: "yo"] = "hey" try await ProcessExecutor.runLogOutput( @@ -609,7 +618,8 @@ final class IntegrationTests: XCTestCase { func testBasicRunMethodWorks() async throws { try await ProcessExecutor.run( group: self.group, - executable: "/bin/dd", ["if=/dev/zero", "bs=\(1024 * 1024)", "count=100"], + executable: "/bin/dd", + ["if=/dev/zero", "bs=\(1024 * 1024)", "count=100"], standardInput: EOFSequence(), logger: self.logger ).throwIfNonZero() @@ -618,7 +628,8 @@ final class IntegrationTests: XCTestCase { func testCollectJustStandardOutput() async throws { let allInfo = try await ProcessExecutor.runCollectingOutput( group: self.group, - executable: "/bin/dd", ["if=/dev/zero", "bs=\(1024 * 1024)", "count=1"], + executable: "/bin/dd", + ["if=/dev/zero", "bs=\(1024 * 1024)", "count=1"], standardInput: EOFSequence(), collectStandardOutput: true, collectStandardError: false, @@ -633,7 +644,8 @@ final class IntegrationTests: XCTestCase { func testCollectJustStandardError() async throws { let allInfo = try await ProcessExecutor.runCollectingOutput( group: self.group, - executable: "/bin/sh", ["-c", "/bin/dd >&2 if=/dev/zero bs=\(1024 * 1024) count=1 status=none"], + executable: "/bin/sh", + ["-c", "/bin/dd >&2 if=/dev/zero bs=\(1024 * 1024) count=1 status=none"], standardInput: EOFSequence(), collectStandardOutput: false, collectStandardError: true, @@ -648,7 +660,8 @@ final class IntegrationTests: XCTestCase { func testCollectNothing() async throws { let allInfo = try await ProcessExecutor.runCollectingOutput( group: self.group, - executable: "/bin/sh", ["-c", "/bin/dd >&2 if=/dev/zero bs=\(1024 * 1024) count=100 status=none"], + executable: "/bin/sh", + ["-c", "/bin/dd >&2 if=/dev/zero bs=\(1024 * 1024) count=100 status=none"], standardInput: EOFSequence(), collectStandardOutput: false, collectStandardError: false, @@ -686,7 +699,8 @@ final class IntegrationTests: XCTestCase { do { let result = try await ProcessExecutor.runCollectingOutput( group: self.group, - executable: "/bin/dd", ["if=/dev/zero", "bs=\(1024 * 1024)", "count=1"], + executable: "/bin/dd", + ["if=/dev/zero", "bs=\(1024 * 1024)", "count=1"], standardInput: EOFSequence(), collectStandardOutput: true, collectStandardError: false, @@ -757,15 +771,16 @@ final class IntegrationTests: XCTestCase { } catch { XCTAssertEqual(NSCocoaErrorDomain, (error as NSError).domain) #if canImport(Darwin) - // https://github.com/apple/swift-corelibs-foundation/issues/4810 - XCTAssertEqual(NSFileNoSuchFileError, (error as NSError).code) + // https://github.com/apple/swift-corelibs-foundation/issues/4810 + XCTAssertEqual(NSFileNoSuchFileError, (error as NSError).code) #endif } } func testAPIsWithoutELGOrLoggerArguments() async throws { let exe = ProcessExecutor( - executable: "/bin/sh", ["-c", "true"], + executable: "/bin/sh", + ["-c", "true"], standardInput: EOFSequence(), standardOutput: .discard, standardError: .discard @@ -773,7 +788,8 @@ final class IntegrationTests: XCTestCase { try await exe.run().throwIfNonZero() try await ProcessExecutor.run( - executable: "/bin/sh", ["-c", "true"], + executable: "/bin/sh", + ["-c", "true"], standardInput: EOFSequence() ).throwIfNonZero() @@ -803,7 +819,8 @@ final class IntegrationTests: XCTestCase { func testAPIsWithoutELGStandardInputOrLoggerArguments() async throws { let exe = ProcessExecutor( - executable: "/bin/sh", ["-c", "true"], + executable: "/bin/sh", + ["-c", "true"], standardOutput: .discard, standardError: .discard ) @@ -811,23 +828,23 @@ final class IntegrationTests: XCTestCase { let exeStream = ProcessExecutor(executable: "/bin/sh", ["-c", "true"]) #if compiler(>=5.8) - async let stdout = Array(exeStream.standardOutput) - async let stderr = Array(exeStream.standardError) + async let stdout = Array(exeStream.standardOutput) + async let stderr = Array(exeStream.standardError) #else - async let stdout = { - var chunks: [ByteBuffer] = [] - for try await chunk in await exeStream.standardOutput { - chunks.append(chunk) - } - return chunks - }() - async let stderr = { - var chunks: [ByteBuffer] = [] - for try await chunk in await exeStream.standardError { - chunks.append(chunk) - } - return chunks - }() + async let stdout = { + var chunks: [ByteBuffer] = [] + for try await chunk in await exeStream.standardOutput { + chunks.append(chunk) + } + return chunks + }() + async let stderr = { + var chunks: [ByteBuffer] = [] + for try await chunk in await exeStream.standardError { + chunks.append(chunk) + } + return chunks + }() #endif try await exeStream.run().throwIfNonZero() let out = try await stdout @@ -863,7 +880,9 @@ final class IntegrationTests: XCTestCase { XCTAssertNoThrow(try FileManager.default.removeItem(at: tempDir)) } - for (stdoutMode, stderrMode) in [("shared", "shared"), ("shared", "owned"), ("owned", "shared")] { + for (stdoutMode, stderrMode) in [ + ("shared", "shared"), ("shared", "owned"), ("owned", "shared"), + ] { let filePath = tempDir.appendingPathComponent("file-\(stdoutMode)-\(stderrMode)") let fd = try FileDescriptor.open( .init(filePath.path.removingPercentEncoding!), @@ -892,14 +911,14 @@ final class IntegrationTests: XCTestCase { } #if canImport(Darwin) - let command = - "for o in 1 2; do i=1000; while [ $i -gt 0 ]; do echo $o >&$o; i=$(( $i - 1 )); done & done; wait" + let command = + "for o in 1 2; do i=1000; while [ $i -gt 0 ]; do echo $o >&$o; i=$(( $i - 1 )); done & done; wait" #else - // workaround for - // https://github.com/apple/swift-corelibs-foundation/issues/4772 - // which causes `SIGCHLD` being blocked in the shell so it can't wait for its children :| - let command = - "for o in 1 2; do i=1000; while [ $i -gt 0 ]; do echo $o >&$o; i=$(( $i - 1 )); done & done; sleep 10" + // workaround for + // https://github.com/apple/swift-corelibs-foundation/issues/4772 + // which causes `SIGCHLD` being blocked in the shell so it can't wait for its children :| + let command = + "for o in 1 2; do i=1000; while [ $i -gt 0 ]; do echo $o >&$o; i=$(( $i - 1 )); done & done; sleep 10" #endif let exe = ProcessExecutor( @@ -959,7 +978,8 @@ final class IntegrationTests: XCTestCase { ) { group in group.addTask { [logger = self.logger!] in try await ProcessExecutor.run( - executable: "/bin/sleep", ["100000"], + executable: "/bin/sleep", + ["100000"], logger: logger ) } @@ -1020,71 +1040,71 @@ final class IntegrationTests: XCTestCase { } #if os(macOS) - // This test will hang on anything that uses swift-corelibs-foundation because of - // https://github.com/apple/swift-corelibs-foundation/issues/4795 - // Foundation.Process on Linux doesn't correctly detect when child process dies (creating zombie processes) - func testCanDealWithRunawayChildProcesses() async throws { - self.logger = Logger(label: "x") - self.logger.logLevel = .info - let p = ProcessExecutor( - executable: "/bin/bash", - [ - "-c", - """ - set -e - /usr/bin/yes "Runaway process from \(#function), please file a swift-sdk-generator bug." > /dev/null & - child_pid=$! - trap "echo >&2 killing $child_pid; kill -KILL $child_pid" INT - echo "$child_pid" # communicate the child pid to our parent - exec >&- # close stdout - echo "waiting for $child_pid" >&2 - wait - """, - ], - standardError: .discard, - teardownSequence: [ - .sendSignal(SIGINT, allowedTimeToExitNS: 10_000_000_000), - ], - logger: self.logger - ) - - try await withThrowingTaskGroup(of: pid_t?.self) { group in - group.addTask { - let result = try await p.run() - XCTAssertEqual(.exit(128 + SIGINT), result) - return nil - } + // This test will deadlock on anything that uses swift-corelibs-foundation because of + // https://github.com/apple/swift-corelibs-foundation/issues/4795 + // Foundation.Process on Linux doesn't correctly detect when child process dies (creating zombie processes) + func testCanDealWithRunawayChildProcesses() async throws { + self.logger = Logger(label: "x") + self.logger.logLevel = .info + let p = ProcessExecutor( + executable: "/bin/bash", + [ + "-c", + """ + set -e + /usr/bin/yes "Runaway process from \(#function), please file a swift-sdk-generator bug." > /dev/null & + child_pid=$! + trap "echo >&2 killing $child_pid; kill -KILL $child_pid" INT + echo "$child_pid" # communicate the child pid to our parent + exec >&- # close stdout + echo "waiting for $child_pid" >&2 + wait + """, + ], + standardError: .discard, + teardownSequence: [ + .sendSignal(SIGINT, allowedTimeToExitNS: 10_000_000_000) + ], + logger: self.logger + ) - group.addTask { - let pidString = try await String(buffer: p.standardOutput.pullAllOfIt()) - guard let pid = pid_t(pidString.dropLast()) else { - XCTFail("couldn't get pid from \(pidString)") + try await withThrowingTaskGroup(of: pid_t?.self) { group in + group.addTask { + let result = try await p.run() + XCTAssertEqual(.exit(128 + SIGINT), result) return nil } - return pid - } - let maybePid = try await group.next()! - let pid = try XCTUnwrap(maybePid) - group.cancelAll() - try await group.waitForAll() + group.addTask { + let pidString = try await String(buffer: p.standardOutput.pullAllOfIt()) + guard let pid = pid_t(pidString.dropLast()) else { + XCTFail("couldn't get pid from \(pidString)") + return nil + } + return pid + } - // Let's check that the subprocess (/usr/bin/yes) of our subprocess (/bin/bash) is actually dead - // This is a tiny bit racy because the pid isn't immediately invalidated, so let's allow a few failures - for attempt in 0 ..< .max { - let killRet = kill(pid, 0) - let errnoCode = errno - guard killRet == -1 || attempt > 5 else { - logger.error("kill didn't fail on attempt \(attempt), trying again...") - usleep(100_000) - continue + let maybePid = try await group.next()! + let pid = try XCTUnwrap(maybePid) + group.cancelAll() + try await group.waitForAll() + + // Let's check that the subprocess (/usr/bin/yes) of our subprocess (/bin/bash) is actually dead + // This is a tiny bit racy because the pid isn't immediately invalidated, so let's allow a few failures + for attempt in 0 ..< .max { + let killRet = kill(pid, 0) + let errnoCode = errno + guard killRet == -1 || attempt > 5 else { + logger.error("kill didn't fail on attempt \(attempt), trying again...") + usleep(100_000) + continue + } + XCTAssertEqual(-1, killRet) + XCTAssertEqual(ESRCH, errnoCode) + break } - XCTAssertEqual(-1, killRet) - XCTAssertEqual(ESRCH, errnoCode) - break - } + } } - } #endif func testShutdownSequenceWorks() async throws { @@ -1115,8 +1135,8 @@ final class IntegrationTests: XCTestCase { group.addTask { let result = try await p.run() #if os(macOS) - // won't work on SCLF: https://github.com/apple/swift-corelibs-foundation/issues/4772 - XCTAssertEqual(.exit(3), result) + // won't work on SCLF: https://github.com/apple/swift-corelibs-foundation/issues/4772 + XCTAssertEqual(.exit(3), result) #endif } var allLines: [String] = [] @@ -1128,8 +1148,8 @@ final class IntegrationTests: XCTestCase { } try await group.waitForAll() #if os(macOS) - // won't work on SCLF: https://github.com/apple/swift-corelibs-foundation/issues/4772 - XCTAssertEqual(["OK", "saw SIGQUIT", "saw SIGTERM", "saw SIGINT"], allLines) + // won't work on SCLF: https://github.com/apple/swift-corelibs-foundation/issues/4772 + XCTAssertEqual(["OK", "saw SIGQUIT", "saw SIGTERM", "saw SIGINT"], allLines) #endif } } diff --git a/Tests/GeneratorEngineTests/EngineTests.swift b/Tests/GeneratorEngineTests/EngineTests.swift index 2e1a90b..c3851e7 100644 --- a/Tests/GeneratorEngineTests/EngineTests.swift +++ b/Tests/GeneratorEngineTests/EngineTests.swift @@ -10,18 +10,24 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.Data import Crypto -@testable import Helpers +import XCTest + +import struct Foundation.Data import struct Logging.Logger import struct SystemPackage.FilePath -import XCTest + +@testable import Helpers private let encoder = JSONEncoder() private let decoder = JSONDecoder() -private extension AsyncFileSystem { - func read(_ path: FilePath, bufferLimit: Int = 10 * 1024 * 1024, as: V.Type) async throws -> V { +extension AsyncFileSystem { + fileprivate func read( + _ path: FilePath, + bufferLimit: Int = 10 * 1024 * 1024, + as: V.Type + ) async throws -> V { let data = try await self.withOpenReadableFile(path) { var data = Data() for try await chunk in try await $0.read() { @@ -35,7 +41,7 @@ private extension AsyncFileSystem { return try decoder.decode(V.self, from: data) } - func write(_ path: FilePath, _ value: some Encodable) async throws { + fileprivate func write(_ path: FilePath, _ value: some Encodable) async throws { let data = try encoder.encode(value) try await self.withOpenWritableFile(path) { fileHandle in try await fileHandle.write(data) @@ -116,7 +122,7 @@ final class EngineTests: XCTestCase { let engine = QueryEngine( MockFileSystem(), Logger(label: "engine-tests") -// cacheLocation: .memory + // cacheLocation: .memory ) var resultPath = try await engine[Expression(x: 1, y: 2)].path @@ -172,9 +178,11 @@ final class EngineTests: XCTestCase { func testQueryEncoding() throws { let item = MyItem( remoteURL: URL( - string: "https://download.swift.org/swift-5.9.2-release/ubuntu2204-aarch64/swift-5.9.2-RELEASE/swift-5.9.2-RELEASE-ubuntu22.04-aarch64.tar.gz" + string: + "https://download.swift.org/swift-5.9.2-release/ubuntu2204-aarch64/swift-5.9.2-RELEASE/swift-5.9.2-RELEASE-ubuntu22.04-aarch64.tar.gz" )!, - localPath: "/Users/katei/ghq/github.com/apple/swift-sdk-generator/Artifacts/target_swift_5.9.2-RELEASE_aarch64-unknown-linux-gnu.tar.gz", + localPath: + "/Users/katei/ghq/github.com/apple/swift-sdk-generator/Artifacts/target_swift_5.9.2-RELEASE_aarch64-unknown-linux-gnu.tar.gz", isPrebuilt: true ) func hashValue(of key: some CacheKey) throws -> SHA256Digest { diff --git a/Tests/HelpersTests/ThrowingDeferTests.swift b/Tests/HelpersTests/ThrowingDeferTests.swift index 2dc530e..03a0d3f 100644 --- a/Tests/HelpersTests/ThrowingDeferTests.swift +++ b/Tests/HelpersTests/ThrowingDeferTests.swift @@ -56,11 +56,13 @@ final class ThrowingDeferTests: XCTestCase { let workError = EquatableError() var didRunCleanup = false - XCTAssertThrowsError(try withThrowing { - throw workError - } defer: { - didRunCleanup = true - }) { + XCTAssertThrowsError( + try withThrowing { + throw workError + } defer: { + didRunCleanup = true + } + ) { XCTAssertTrue($0 is EquatableError) XCTAssertEqual($0 as? EquatableError, workError) } @@ -71,11 +73,13 @@ final class ThrowingDeferTests: XCTestCase { var didRunWork = false let cleanupError = EquatableError() - XCTAssertThrowsError(try withThrowing { - didRunWork = true - } defer: { - throw cleanupError - }) { + XCTAssertThrowsError( + try withThrowing { + didRunWork = true + } defer: { + throw cleanupError + } + ) { XCTAssertTrue($0 is EquatableError) XCTAssertEqual($0 as? EquatableError, cleanupError) } @@ -87,12 +91,14 @@ final class ThrowingDeferTests: XCTestCase { let workError = EquatableError() let cleanupError = EquatableError() - XCTAssertThrowsError(try withThrowing { - didRunWork = true - throw workError - } defer: { - throw cleanupError - }) { + XCTAssertThrowsError( + try withThrowing { + didRunWork = true + throw workError + } defer: { + throw cleanupError + } + ) { XCTAssertTrue($0 is EquatableError) XCTAssertEqual($0 as? EquatableError, cleanupError) } diff --git a/Tests/MacrosTests/MacrosTests.swift b/Tests/MacrosTests/MacrosTests.swift index f9a0b46..c8c457f 100644 --- a/Tests/MacrosTests/MacrosTests.swift +++ b/Tests/MacrosTests/MacrosTests.swift @@ -1,6 +1,6 @@ //===----------------------------------------------------------------------===// // -// This source file is part of the Swift.org open source project +// This source file is part of the Swift open source project // // Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception @@ -16,7 +16,9 @@ import SwiftSyntaxMacrosTestSupport import XCTest final class MacrosTests: XCTestCase { - private let macros: [String: Macro.Type] = ["CacheKey": CacheKeyMacro.self, "Query": QueryMacro.self] + private let macros: [String: Macro.Type] = [ + "CacheKey": CacheKeyMacro.self, "Query": QueryMacro.self, + ] func testCacheKeyDerived() { assertMacroExpansion( @@ -34,34 +36,34 @@ final class MacrosTests: XCTestCase { } """, expandedSource: """ - struct Message { - let text: String - let sender: String - } - struct Q { - let number: Int - let text: String - } + struct Message { + let text: String + let sender: String + } + struct Q { + let number: Int + let text: String + } - extension Message: CacheKeyProtocol { - func hash(with hashFunction: inout some HashFunction) { - String(reflecting: Self.self).hash(with: &hashFunction) - text.hash(with: &hashFunction) - sender.hash(with: &hashFunction) + extension Message: CacheKeyProtocol { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + text.hash(with: &hashFunction) + sender.hash(with: &hashFunction) + } } - } - extension Q: QueryProtocol { - } + extension Q: QueryProtocol { + } - extension Q: CacheKeyProtocol { - func hash(with hashFunction: inout some HashFunction) { - String(reflecting: Self.self).hash(with: &hashFunction) - number.hash(with: &hashFunction) - text.hash(with: &hashFunction) + extension Q: CacheKeyProtocol { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + number.hash(with: &hashFunction) + text.hash(with: &hashFunction) + } } - } - """, + """, macros: self.macros, indentationWidth: .spaces(2) ) diff --git a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift index 9202c2a..893d8c7 100644 --- a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift +++ b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift @@ -11,9 +11,10 @@ //===----------------------------------------------------------------------===// import Logging -@testable import SwiftSDKGenerator import XCTest +@testable import SwiftSDKGenerator + final class ArchitectureMappingTests: XCTestCase { let logger = Logger(label: "ArchitectureMappingTests") @@ -39,12 +40,12 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple, targetTriple: Triple, - artifactID: String, // Base name of the generated bundle - hostLLVMDownloadURL: String, // URL of the host LLVM package - targetSwiftDownloadURL: String, // URL of the target Swift SDK + artifactID: String, // Base name of the generated bundle + hostLLVMDownloadURL: String, // URL of the host LLVM package + targetSwiftDownloadURL: String, // URL of the target Swift SDK - artifactBundlePathSuffix: String, // Path to the generated bundle - sdkDirPathSuffix: String // Path of the SDK within the bundle + artifactBundlePathSuffix: String, // Path to the generated bundle + sdkDirPathSuffix: String // Path of the SDK within the bundle ) async throws { let recipe = try LinuxRecipe( targetTriple: targetTriple, @@ -130,8 +131,10 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple("x86_64-apple-macosx13"), targetTriple: Triple("x86_64-unknown-linux-gnu"), artifactID: "5.8-RELEASE_ubuntu_jammy_x86_64", - hostLLVMDownloadURL: "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", - targetSwiftDownloadURL: "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", + hostLLVMDownloadURL: + "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", + targetSwiftDownloadURL: + "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_x86_64.artifactbundle", sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_x86_64/x86_64-unknown-linux-gnu/ubuntu-jammy.sdk" ) @@ -143,10 +146,13 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple("x86_64-apple-macosx13"), targetTriple: Triple("aarch64-unknown-linux-gnu"), artifactID: "5.8-RELEASE_ubuntu_jammy_aarch64", - hostLLVMDownloadURL: "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", - targetSwiftDownloadURL: "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", + hostLLVMDownloadURL: + "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", + targetSwiftDownloadURL: + "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_aarch64.artifactbundle", - sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" + sdkDirPathSuffix: + "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" ) } @@ -156,10 +162,13 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple("arm64-apple-macosx13"), targetTriple: Triple("aarch64-unknown-linux-gnu"), artifactID: "5.8-RELEASE_ubuntu_jammy_aarch64", - hostLLVMDownloadURL: "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", - targetSwiftDownloadURL: "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", + hostLLVMDownloadURL: + "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", + targetSwiftDownloadURL: + "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_aarch64.artifactbundle", - sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" + sdkDirPathSuffix: + "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" ) } @@ -169,8 +178,10 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple("arm64-apple-macosx13"), targetTriple: Triple("x86_64-unknown-linux-gnu"), artifactID: "5.8-RELEASE_ubuntu_jammy_x86_64", - hostLLVMDownloadURL: "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", - targetSwiftDownloadURL: "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", + hostLLVMDownloadURL: + "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", + targetSwiftDownloadURL: + "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_x86_64.artifactbundle", sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_x86_64/x86_64-unknown-linux-gnu/ubuntu-jammy.sdk" ) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index ffbfc26..9a9222f 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -18,7 +18,11 @@ import XCTest @testable import SwiftSDKGenerator extension FileManager { - func withTemporaryDirectory(logger: Logger, cleanup: Bool = true, body: (URL) async throws -> T) async throws -> T { + func withTemporaryDirectory( + logger: Logger, + cleanup: Bool = true, + body: (URL) async throws -> T + ) async throws -> T { // Create a temporary directory using a UUID. Throws if the directory already exists. // The docs suggest using FileManager.url(for: .itemReplacementDirectory, ...) to create a temporary directory, // but on Linux the directory name contains spaces, which means we need to be careful to quote it everywhere: @@ -33,15 +37,13 @@ extension FileManager { try createDirectory(at: temporaryDirectory, withIntermediateDirectories: false) defer { - // Best effort cleanup. - do { - if cleanup { - try removeItem(at: temporaryDirectory) - logger.info("Removed temporary directory") - } else { - logger.info("Keeping temporary directory") - } - } catch {} + // Best effort cleanup. + if cleanup { + try? removeItem(at: temporaryDirectory) + logger.info("Removed temporary directory") + } else { + logger.info("Keeping temporary directory") + } } logger.info("Created temporary directory") @@ -53,12 +55,14 @@ extension FileManager { // This takes a lock on `.build`, but if the tests are being run by `swift test` the outer Swift Package Manager // instance will already hold this lock, causing the test to deadlock. We can work around this by giving // the `swift run swift-sdk-generator` instance its own scratch directory. -func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: String) async throws -> String { +func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: String) + async throws -> String +{ var logger = logger logger[metadataKey: "runArguments"] = "\"\(runArguments)\"" logger[metadataKey: "scratchPath"] = "\(scratchPath)" - logger.info("Building SDK") + logger.info("Building Swift SDK") var packageDirectory = FilePath(#filePath) packageDirectory.removeLastComponent() @@ -67,27 +71,31 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: let generatorOutput = try await Shell.readStdout( "cd \(packageDirectory) && swift run --scratch-path \"\(scratchPath)\" swift-sdk-generator make-linux-sdk \(runArguments)" ) - logger.info("Finished building SDK") + logger.info("Finished building Swift SDK") - let installCommand = try XCTUnwrap(generatorOutput.split(separator: "\n").first { - $0.contains("swift experimental-sdk install") - }) + let installCommand = try XCTUnwrap( + generatorOutput.split(separator: "\n").first { + $0.contains("swift experimental-sdk install") + } + ) let bundleName = try XCTUnwrap( FilePath(String(XCTUnwrap(installCommand.split(separator: " ").last))).components.last ).stem logger[metadataKey: "bundleName"] = "\(bundleName)" - logger.info("Checking installed SDKs") - let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components(separatedBy: "\n") + logger.info("Checking installed Swift SDKs") + let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components( + separatedBy: "\n" + ) // Make sure this bundle hasn't been installed already. if installedSDKs.contains(bundleName) { - logger.info("Removing existing SDK") + logger.info("Removing existing Swift SDK") try await Shell.run("swift experimental-sdk remove \(bundleName)") } - logger.info("Installing new SDK") + logger.info("Installing new Swift SDK") let installOutput = try await Shell.readStdout(String(installCommand)) XCTAssertTrue(installOutput.contains("successfully installed")) @@ -115,8 +123,12 @@ final class RepeatedBuildTests: XCTestCase { private let logger = Logger(label: "swift-sdk-generator") func testRepeatedSDKBuilds() async throws { - if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { - throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") + if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") + || ProcessInfo.processInfo.environment.keys.contains("GITHUB_ACTIONS") + { + throw XCTSkip( + "EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145" + ) } var logger = logger @@ -127,15 +139,21 @@ final class RepeatedBuildTests: XCTestCase { var possibleArguments = ["--host-toolchain"] do { try await Shell.run("docker ps") - possibleArguments.append("--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9") + possibleArguments.append( + "--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9" + ) } catch { - self.logger.warning("Docker CLI does not seem to be working, skipping tests that involve Docker.") + self.logger.warning( + "Docker CLI does not seem to be working, skipping tests that involve Docker." + ) } for runArguments in possibleArguments { if runArguments.contains("rhel") { // Temporarily skip the RHEL-based SDK. XCTSkip() is not suitable as it would skipping the entire test case - logger.warning("RHEL-based SDKs currently do not work with Swift 6.0: https://github.com/swiftlang/swift-sdk-generator/issues/138") + logger.warning( + "RHEL-based SDKs currently do not work with Swift 6.0: https://github.com/swiftlang/swift-sdk-generator/issues/138" + ) continue } @@ -151,10 +169,15 @@ final class RepeatedBuildTests: XCTestCase { struct SDKConfiguration { var swiftVersion: String var linuxDistributionName: String + var linuxDistributionVersion: String var architecture: String var withDocker: Bool + var containerImageSuffix: String? - var bundleName: String { "\(linuxDistributionName)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" } + var bundleName: String { + let sdkPrefix = containerImageSuffix ?? "\(linuxDistributionName)_\(linuxDistributionVersion)" + return "\(sdkPrefix)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" + } func withDocker(_ enabled: Bool = true) -> SDKConfiguration { var res = self @@ -162,6 +185,18 @@ struct SDKConfiguration { return res } + func withContainerImageSuffix(_ containerImageSuffix: String) -> SDKConfiguration { + var res = self + res.containerImageSuffix = containerImageSuffix + return res + } + + func withLinuxDistributionVersion(_ version: String) -> SDKConfiguration { + var res = self + res.linuxDistributionVersion = version + return res + } + func withArchitecture(_ arch: String) -> SDKConfiguration { var res = self res.architecture = arch @@ -174,15 +209,23 @@ struct SDKConfiguration { } var sdkGeneratorArguments: String { + // Build the container image tag + var containerImage: String? = nil + if let containerImageSuffix { + containerImage = "swift:\(swiftVersion)-\(containerImageSuffix)" + } + return [ "--sdk-name \(bundleName)", "--host-toolchain", withDocker ? "--with-docker" : nil, + containerImage != nil ? "--from-container-image" : nil, containerImage, "--swift-version \(swiftVersion)-RELEASE", testLinuxSwiftSDKs ? "--host \(hostArch!)-unknown-linux-gnu" : nil, "--target \(architecture)-unknown-linux-gnu", - "--linux-distribution-name \(linuxDistributionName)" - ].compactMap{ $0 }.joined(separator: " ") + "--linux-distribution-name \(linuxDistributionName)", + "--linux-distribution-version \(linuxDistributionVersion)", + ].compactMap { $0 }.joined(separator: " ") } } @@ -198,10 +241,15 @@ var testLinuxSwiftSDKs: Bool { ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_TEST_LINUX_SWIFT_SDKS") } -func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) async throws { +func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) + async throws +{ let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") let testPackageDir = FilePath(testPackageURL.path) - try FileManager.default.createDirectory(atPath: testPackageDir.string, withIntermediateDirectories: true) + try FileManager.default.createDirectory( + atPath: testPackageDir.string, + withIntermediateDirectories: true + ) logger.info("Creating test project \(testPackageDir)") try await Shell.run("swift package --package-path \(testPackageDir) init --type executable") @@ -226,7 +274,9 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD // that contains each Swift-supported Linux distribution. This way we can validate that each // distribution is capable of building using the Linux Swift SDK. if testLinuxSwiftSDKs { - let swiftContainerVersions = ["focal", "jammy", "noble", "fedora39", "rhel-ubi9", "amazonlinux2", "bookworm"] + let swiftContainerVersions = [ + "focal", "jammy", "noble", "fedora39", "rhel-ubi9", "amazonlinux2", "bookworm", + ] for containerVersion in swiftContainerVersions { logger.info("Building test project in 6.0-\(containerVersion) container") buildOutput = try await Shell.readStdout( @@ -240,7 +290,9 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD XCTAssertTrue(buildOutput.contains("Build complete!")) logger.info("Test project built successfully") - logger.info("Building test project in 6.0-\(containerVersion) container with static-swift-stdlib") + logger.info( + "Building test project in 6.0-\(containerVersion) container with static-swift-stdlib" + ) buildOutput = try await Shell.readStdout( """ docker run --rm -v \(testPackageDir):/src \ @@ -275,8 +327,12 @@ func buildTestcases(config: SDKConfiguration) async throws { var logger = Logger(label: "EndToEndTests") logger[metadataKey: "testcase"] = "testPackageInitExecutable" - if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { - throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") + if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") + || ProcessInfo.processInfo.environment.keys.contains("GITHUB_ACTIONS") + { + throw XCTSkip( + "EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145" + ) } if config.withDocker { @@ -288,26 +344,45 @@ func buildTestcases(config: SDKConfiguration) async throws { } let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - try await buildSDK(logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) + try await buildSDK( + logger, + scratchPath: tempDir.path, + withArguments: config.sdkGeneratorArguments + ) } - logger.info("Built SDK") + logger.info("Built Swift SDK") + + // Cleanup + func cleanupSDK() async { + logger.info("Removing Swift SDK to clean up...") + try? await Shell.run("swift experimental-sdk remove \(bundleName)") + } for testcase in testcases { - try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - try await buildTestcase(logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir) + do { + try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + try await buildTestcase( + logger, + testcase: testcase, + bundleName: bundleName, + tempDir: tempDir + ) + } + } catch { + await cleanupSDK() + throw error } } - // Cleanup - logger.info("Removing SDK to cleanup...") - try await Shell.run("swift experimental-sdk remove \(bundleName)") + await cleanupSDK() } final class Swift59_UbuntuEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", linuxDistributionName: "ubuntu", + linuxDistributionVersion: "22.04", architecture: "aarch64", withDocker: false ) @@ -337,6 +412,7 @@ final class Swift510_UbuntuEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.10.1", linuxDistributionName: "ubuntu", + linuxDistributionVersion: "22.04", architecture: "aarch64", withDocker: false ) @@ -366,6 +442,7 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "6.0.3", linuxDistributionName: "ubuntu", + linuxDistributionVersion: "24.04", architecture: "aarch64", withDocker: false ) @@ -391,22 +468,273 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { } } +final class Swift61_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "ubuntu", + linuxDistributionVersion: "24.04", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testJammyAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withLinuxDistributionVersion("22.04") + ) + } + + func testJammyX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withLinuxDistributionVersion("22.04") + ) + } + + func testJammyAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withLinuxDistributionVersion("22.04").withDocker() + ) + } + + func testJammyX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withLinuxDistributionVersion("22.04").withDocker() + ) + } +} + +final class Swift59_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "debian", + linuxDistributionVersion: "11", // we use ubuntu2004 toolchain here + architecture: "aarch64", + withDocker: false + ) + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("aarch64") + ) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("x86_64") + ) + } + + // NOTE: the generator does not support building a Debian 11/Debian 12 Swift SDK with Docker + // for Swift 5.9.x and 5.10 without a pre-built container, so we do not test this here. +} + +final class Swift510_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift60_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.3", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift61_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", architecture: "aarch64", withDocker: true // RHEL-based SDKs can only be built from containers ) func testAarch64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + try await buildTestcases(config: config.withArchitecture("aarch64")) } func testX86_64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) } } @@ -414,18 +742,47 @@ final class Swift510_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.10.1", linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", architecture: "aarch64", withDocker: true // RHEL-based SDKs can only be built from containers ) func testAarch64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + try await buildTestcases(config: config.withArchitecture("aarch64")) } func testX86_64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testFedora39Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39") + ) + } + + func testFedora39X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39") + ) } } @@ -433,17 +790,80 @@ final class Swift60_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "6.0.3", linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", architecture: "aarch64", withDocker: true // RHEL-based SDKs can only be built from containers ) func testAarch64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + try await buildTestcases(config: config.withArchitecture("aarch64")) } func testX86_64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testFedora39Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39") + ) + } + + func testFedora39X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39") + ) + } +} + +final class Swift61_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) } } diff --git a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift index 9baf7de..5074efd 100644 --- a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift +++ b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift @@ -16,6 +16,12 @@ import XCTest @testable import SwiftSDKGenerator +#if canImport(FoundationEssentials) + import FoundationEssentials +#else + import Foundation +#endif + final class SwiftSDKGeneratorMetadataTests: XCTestCase { let logger = Logger(label: "SwiftSDKGeneratorMetadataTests") @@ -35,7 +41,7 @@ final class SwiftSDKGeneratorMetadataTests: XCTestCase { bundleVersion: "0.0.3", targetTriple: Triple("armv7-unknown-linux-gnueabihf"), expectedCanonicalName: "armv7-swift-linux-gnueabihf" - ) + ), ] for testCase in testCases { @@ -54,19 +60,44 @@ final class SwiftSDKGeneratorMetadataTests: XCTestCase { // Make sure the file exists let sdkSettingsFile = sdkDirPath.appending("SDKSettings.json") - let fileExists = await sdk.doesFileExist(at: sdkSettingsFile) + var fileExists = await sdk.doesFileExist(at: sdkSettingsFile) XCTAssertTrue(fileExists) // Read back file, make sure it contains the expected data - let data = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) - XCTAssertNotNil(data) - XCTAssertTrue(data!.contains(testCase.bundleVersion)) - XCTAssertTrue(data!.contains("(\(testCase.targetTriple.archName))")) - XCTAssertTrue(data!.contains(linuxDistribution.description)) - XCTAssertTrue(data!.contains(testCase.expectedCanonicalName)) + let maybeString = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) + let string = try XCTUnwrap(maybeString) + XCTAssertTrue(string.contains(testCase.bundleVersion)) + XCTAssertTrue(string.contains("(\(testCase.targetTriple.archName))")) + XCTAssertTrue(string.contains(linuxDistribution.description)) + XCTAssertTrue(string.contains(testCase.expectedCanonicalName)) // Cleanup try await sdk.removeFile(at: sdkSettingsFile) + + try await sdk.createDirectoryIfNeeded(at: sdk.pathsConfiguration.artifactBundlePath) + + // Generate bundle metadata + try await sdk.generateArtifactBundleManifest( + hostTriples: [sdk.targetTriple], + artifacts: ["foo": sdk.pathsConfiguration.artifactBundlePath.appending("bar.json")] + ) + + // Make sure the file exists + let archiveMetadataFile = await sdk.pathsConfiguration.artifactBundlePath.appending("info.json") + fileExists = await sdk.doesFileExist(at: archiveMetadataFile) + XCTAssertTrue(fileExists) + + // Read back file, make sure it contains the expected data + let data = try await sdk.readFile(at: archiveMetadataFile) + let decodedMetadata = try JSONDecoder().decode(ArtifactsArchiveMetadata.self, from: data) + XCTAssertEqual(decodedMetadata.artifacts.count, 1) + for (id, artifact) in decodedMetadata.artifacts { + XCTAssertEqual(id, "foo") + XCTAssertEqual(artifact.variants, [.init(path: "bar.json", supportedTriples: [testCase.targetTriple.triple])]) + } + + // Cleanup + try await sdk.removeFile(at: archiveMetadataFile) } } } diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 436b705..b57807e 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -20,6 +20,7 @@ final class LinuxRecipeTests: XCTestCase { func createRecipe( hostTriple: Triple = Triple("x86_64-unknown-linux-gnu"), + linuxDistribution: LinuxDistribution, swiftVersion: String = "6.0", withDocker: Bool = false, fromContainerImage: String? = nil, @@ -27,13 +28,15 @@ final class LinuxRecipeTests: XCTestCase { targetSwiftPackagePath: String? = nil, includeHostToolchain: Bool = true ) throws -> LinuxRecipe { - try LinuxRecipe( + try LinuxRecipe( targetTriple: Triple("aarch64-unknown-linux-gnu"), hostTriple: hostTriple, - linuxDistribution: .init(name: .ubuntu, version: "22.04"), + linuxDistribution: linuxDistribution, swiftVersion: swiftVersion, - swiftBranch: nil, lldVersion: "", - withDocker: withDocker, fromContainerImage: fromContainerImage, + swiftBranch: nil, + lldVersion: "", + withDocker: withDocker, + fromContainerImage: fromContainerImage, hostSwiftPackagePath: hostSwiftPackagePath, targetSwiftPackagePath: targetSwiftPackagePath, includeHostToolchain: includeHostToolchain, @@ -45,27 +48,45 @@ final class LinuxRecipeTests: XCTestCase { let testCases = [ ( swiftVersion: "5.9.2", + targetTriple: Triple("x86_64-unknown-linux-gnu"), expectedSwiftCompilerOptions: [ "-Xlinker", "-R/usr/lib/swift/linux/", - "-Xclang-linker", "--ld-path=ld.lld" + "-Xclang-linker", "--ld-path=ld.lld", ], expectedLinkerPath: nil ), ( swiftVersion: "6.0.2", + targetTriple: Triple("aarch64-unknown-linux-gnu"), expectedSwiftCompilerOptions: [ "-Xlinker", "-R/usr/lib/swift/linux/", - "-use-ld=lld" + "-use-ld=lld", ], expectedLinkerPath: "ld.lld" - ) + ), + ( + swiftVersion: "6.0.3", + targetTriple: Triple("armv7-unknown-linux-gnueabihf"), + expectedSwiftCompilerOptions: [ + "-Xlinker", "-R/usr/lib/swift/linux/", + "-use-ld=lld", + "-latomic", + ], + expectedLinkerPath: "ld.lld" + ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { - let recipe = try self.createRecipe(swiftVersion: testCase.swiftVersion) + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + swiftVersion: testCase.swiftVersion + ) var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: Triple("aarch64-unknown-linux-gnu") + toolset: &toolset, + targetTriple: testCase.targetTriple, + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, testCase.expectedSwiftCompilerOptions) XCTAssertEqual(toolset.linker?.path, testCase.expectedLinkerPath) @@ -75,23 +96,35 @@ final class LinuxRecipeTests: XCTestCase { } func testToolOptionsForPreinstalledSdk() throws { - let recipe = try self.createRecipe(includeHostToolchain: false) + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + includeHostToolchain: false + ) var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: Triple("x86_64-unknown-linux-gnu") + toolset: &toolset, + targetTriple: Triple("x86_64-unknown-linux-gnu"), + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.rootPath, nil) - XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, [ - "-Xlinker", "-R/usr/lib/swift/linux/", - "-use-ld=lld" - ]) + XCTAssertEqual( + toolset.swiftCompiler?.extraCLIOptions, + [ + "-Xlinker", "-R/usr/lib/swift/linux/", + "-use-ld=lld", + ] + ) XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ["-lstdc++"]) XCTAssertEqual(toolset.librarian?.path, "llvm-ar") XCTAssert(toolset.linker == nil) } func runItemsToDownloadTestCase( - recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, includesHostSwift: Bool + recipe: LinuxRecipe, + includesHostLLVM: Bool, + includesTargetSwift: Bool, + includesHostSwift: Bool ) throws { let pathsConfiguration = PathsConfiguration( sourceRoot: ".", @@ -105,9 +138,15 @@ final class LinuxRecipeTests: XCTestCase { pathsConfiguration ) let itemsToDownload = recipe.itemsToDownload(from: downloadableArtifacts) - let foundHostLLVM = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL }) - let foundTargetSwift = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL }) - let foundHostSwift = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.hostSwift.remoteURL }) + let foundHostLLVM = itemsToDownload.contains(where: { + $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL + }) + let foundTargetSwift = itemsToDownload.contains(where: { + $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL + }) + let foundHostSwift = itemsToDownload.contains(where: { + $0.remoteURL == downloadableArtifacts.hostSwift.remoteURL + }) // If this is a Linux host, we do not download LLVM XCTAssertEqual(foundHostLLVM, includesHostLLVM) @@ -117,25 +156,116 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForMacOSHost() throws { let hostTriple = Triple("x86_64-apple-macos") - let testCases: [(recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, includesHostSwift: Bool)] = [ + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") + let testCases: + [( + recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, + includesHostSwift: Bool + )] = [ + ( + // Remote tarballs on Swift < 6.0 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), + includesHostLLVM: true, + includesTargetSwift: true, + includesHostSwift: true + ), + ( + // Remote tarballs on Swift >= 6.0 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), + includesHostLLVM: false, + includesTargetSwift: true, + includesHostSwift: true + ), + ( + // Remote target tarball with preinstalled toolchain + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.9", + includeHostToolchain: false + ), + includesHostLLVM: false, + includesTargetSwift: true, + includesHostSwift: false + ), + ( + // Local packages with Swift < 6.0 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10", + hostSwiftPackagePath: "/path/to/host/swift", + targetSwiftPackagePath: "/path/to/target/swift" + ), + includesHostLLVM: true, + includesTargetSwift: false, + includesHostSwift: false + ), + ( + // Local packages with Swift >= 6.0 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0", + hostSwiftPackagePath: "/path/to/host/swift", + targetSwiftPackagePath: "/path/to/target/swift" + ), + includesHostLLVM: false, + includesTargetSwift: false, + includesHostSwift: false + ), + ] + + for testCase in testCases { + try runItemsToDownloadTestCase( + recipe: testCase.recipe, + includesHostLLVM: testCase.includesHostLLVM, + includesTargetSwift: testCase.includesTargetSwift, + includesHostSwift: testCase.includesHostSwift + ) + } + } + + func testItemsToDownloadForLinuxHost() throws { + let hostTriple = Triple("x86_64-unknown-linux-gnu") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") + let testCases = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), - includesHostLLVM: true, + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), - includesHostLLVM: false, + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesTargetSwift: true, includesHostSwift: true ), ( // Remote target tarball with preinstalled toolchain - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), - includesHostLLVM: false, + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.9", + includeHostToolchain: false + ), includesTargetSwift: true, includesHostSwift: false ), @@ -143,11 +273,11 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" ), - includesHostLLVM: true, includesTargetSwift: false, includesHostSwift: false ), @@ -155,78 +285,106 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" ), - includesHostLLVM: false, includesTargetSwift: false, includesHostSwift: false - ) + ), ] for testCase in testCases { try runItemsToDownloadTestCase( recipe: testCase.recipe, - includesHostLLVM: testCase.includesHostLLVM, + includesHostLLVM: false, // when host is Linux we do not download LLVM includesTargetSwift: testCase.includesTargetSwift, includesHostSwift: testCase.includesHostSwift ) } } - func testItemsToDownloadForLinuxHost() throws { + // Ubuntu toolchains will be selected for Debian 11 and 12 depending on the Swift version + func testItemsToDownloadForDebianTargets() throws { let hostTriple = Triple("x86_64-unknown-linux-gnu") let testCases = [ ( - // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), - includesTargetSwift: true, - includesHostSwift: true + // Debian 11 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu20.04" ), ( - // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), - includesTargetSwift: true, - includesHostSwift: true + // Debian 12 with Swift 5.9 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu22.04" ), ( - // Remote target tarball with preinstalled toolchain - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), - includesTargetSwift: true, - includesHostSwift: false + // Debian 12 with Swift 5.10 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10" + ), + expectedTargetSwift: "ubuntu22.04" ), ( - // Local packages with Swift < 6.0 + // Debian 11 with Swift 6.0 -> ubuntu20.04 recipe: try createRecipe( hostTriple: hostTriple, - swiftVersion: "5.10", - hostSwiftPackagePath: "/path/to/host/swift", - targetSwiftPackagePath: "/path/to/target/swift" + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "6.0" ), - includesTargetSwift: false, - includesHostSwift: false + expectedTargetSwift: "ubuntu20.04" ), ( - // Local packages with Swift >= 6.0 + // Debian 12 with Swift 5.10.1 -> debian12 recipe: try createRecipe( hostTriple: hostTriple, - swiftVersion: "6.0", - hostSwiftPackagePath: "/path/to/host/swift", - targetSwiftPackagePath: "/path/to/target/swift" + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10.1" ), - includesTargetSwift: false, - includesHostSwift: false - ) + expectedTargetSwift: "debian12" + ), + ( + // Debian 12 with Swift 6.0 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "debian12" + ), ] for testCase in testCases { - try runItemsToDownloadTestCase( - recipe: testCase.recipe, - includesHostLLVM: false, // when host is Linux we do not download LLVM - includesTargetSwift: testCase.includesTargetSwift, - includesHostSwift: testCase.includesHostSwift + + let pathsConfiguration = PathsConfiguration( + sourceRoot: ".", + artifactID: "my-sdk", + targetTriple: testCase.recipe.mainTargetTriple + ) + let downloadableArtifacts = try DownloadableArtifacts( + hostTriple: testCase.recipe.mainHostTriple, + targetTriple: testCase.recipe.mainTargetTriple, + testCase.recipe.versionsConfiguration, + pathsConfiguration ) + let itemsToDownload = testCase.recipe.itemsToDownload(from: downloadableArtifacts) + let targetSwiftRemoteURL = itemsToDownload.first(where: { + $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL + })?.remoteURL.absoluteString + + // If this is a Linux host, we do not download LLVM + XCTAssert(targetSwiftRemoteURL!.contains(testCase.expectedTargetSwift)) } } @@ -241,11 +399,16 @@ final class LinuxRecipeTests: XCTestCase { (swiftVersion: "5.9", includeHostToolchain: false, expectedHostTriples: allHostTriples), (swiftVersion: "5.10", includeHostToolchain: false, expectedHostTriples: allHostTriples), (swiftVersion: "6.0", includeHostToolchain: false, expectedHostTriples: nil), - (swiftVersion: "6.0", includeHostToolchain: true, expectedHostTriples: [Triple("x86_64-unknown-linux-gnu")]) + ( + swiftVersion: "6.0", includeHostToolchain: true, + expectedHostTriples: [Triple("x86_64-unknown-linux-gnu")] + ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { let recipe = try createRecipe( + linuxDistribution: linuxDistribution, swiftVersion: testCase.swiftVersion, includeHostToolchain: testCase.includeHostToolchain ) diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift index 71eff42..dff9499 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -17,7 +17,7 @@ import XCTest final class WebAssemblyRecipeTests: XCTestCase { let logger = Logger(label: "WebAssemblyRecipeTests") - + func createRecipe() -> WebAssemblyRecipe { WebAssemblyRecipe( hostSwiftPackage: nil, @@ -32,7 +32,9 @@ final class WebAssemblyRecipeTests: XCTestCase { let recipe = self.createRecipe() var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: Triple("wasm32-unknown-wasi") + toolset: &toolset, + targetTriple: Triple("wasm32-unknown-wasi"), + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, ["-static-stdlib"]) XCTAssertNil(toolset.cCompiler) @@ -40,11 +42,36 @@ final class WebAssemblyRecipeTests: XCTestCase { XCTAssertNil(toolset.linker) } + func testEmbeddedToolOptions() { + let recipe = self.createRecipe() + var toolset = Toolset(rootPath: nil) + recipe.applyPlatformOptions( + toolset: &toolset, + targetTriple: Triple("wasm32-unknown-wasi"), + isForEmbeddedSwift: true + ) + XCTAssertEqual( + toolset.swiftCompiler?.extraCLIOptions, + [ + "-static-stdlib", + "-enable-experimental-feature", "Embedded", "-wmo", + ] + + ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { + ["-Xlinker", $0] + } + ) + XCTAssertEqual(toolset.cCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) + XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) + XCTAssertNil(toolset.linker) + } + func testToolOptionsWithThreads() { let recipe = self.createRecipe() var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: Triple("wasm32-unknown-wasip1-threads") + toolset: &toolset, + targetTriple: Triple("wasm32-unknown-wasip1-threads"), + isForEmbeddedSwift: false ) XCTAssertEqual( toolset.swiftCompiler?.extraCLIOptions, @@ -65,8 +92,11 @@ final class WebAssemblyRecipeTests: XCTestCase { ] XCTAssertEqual(toolset.cCompiler?.extraCLIOptions, ccOptions) XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ccOptions) - XCTAssertEqual(toolset.linker?.extraCLIOptions, [ - "--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", - ]) + XCTAssertEqual( + toolset.linker?.extraCLIOptions, + [ + "--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", + ] + ) } } diff --git a/Utilities/soundness.sh b/Utilities/soundness.sh deleted file mode 100755 index 3ff517d..0000000 --- a/Utilities/soundness.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift open source project -## -## Copyright (c) 2022-2023 Apple Inc. and the Swift project authors -## Licensed under Apache License v2.0 with Runtime Library Exception -## -## See https://swift.org/LICENSE.txt for license information -## See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -## -##===----------------------------------------------------------------------===## - -set -eu -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -function replace_acceptable_years() { - # this needs to replace all acceptable forms with 'YEARS' - sed -e 's/20[12][0123456789]-20[12][0123456789]/YEARS/' -e 's/20[12][0123456789]/YEARS/' -} - -printf "=> Checking for unacceptable language... " -# This greps for unacceptable terminology. The square bracket[s] are so that -# "git grep" doesn't find the lines that greps :). -unacceptable_terms=( - -e blacklis[t] - -e whitelis[t] - -e slav[e] - -e sanit[y] - -e maste[r] -) - -# We have to exclude the code of conduct as it gives examples of unacceptable language. -if git grep --color=never -i "${unacceptable_terms[@]}" -- . > /dev/null; then - printf "\033[0;31mUnacceptable language found.\033[0m\n" - git grep -i "${unacceptable_terms[@]}" -- . - exit 1 -fi -printf "\033[0;32mokay.\033[0m\n" - -printf "=> Checking format... \n" -git diff --name-only | grep ".swift" | while read changed_file; do - printf " * checking ${changed_file}... " - before=$(cat "${changed_file}") - swiftformat "$changed_file" > /dev/null 2>&1 - after=$(cat "${changed_file}") - - if [[ "$before" != "$after" ]]; then - printf "\033[0;31mformatting issues!\033[0m\n" - git --no-pager diff "${changed_file}" - exit 1 - else - printf "\033[0;32mokay.\033[0m\n" - fi -done - -printf "=> Checking license headers... \n" -tmp=$(mktemp /tmp/.swift-package-manager-soundness_XXXXXX) - -for language in swift-or-c bash python; do - printf " * $language... " - declare -a matching_files - declare -a exceptions - expections=( ) - matching_files=( -name '*' ) - case "$language" in - swift-or-c) - exceptions=( -name "Package.swift" -o -path "./Examples/*" -o -path "./Fixtures/*" \ - -o -path "./IntegrationTests/*" -o -path "./Tests/*" -o -path "./Bundles/*" ) - matching_files=( -name '*.swift' -o -name '*.c' -o -name '*.h' ) - cat > "$tmp" <<"EOF" -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) YEARS Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -EOF - ;; - bash) - exceptions=( -path "./Examples/*" -o -path "./Fixtures/*" -o -path "./IntegrationTests/*" \ - -o -path "./Bundles/*" ) - matching_files=( -name '*.sh' ) - cat > "$tmp" <<"EOF" -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift open source project -## -## Copyright (c) YEARS Apple Inc. and the Swift project authors -## Licensed under Apache License v2.0 with Runtime Library Exception -## -## See https://swift.org/LICENSE.txt for license information -## See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -## -##===----------------------------------------------------------------------===## -EOF - ;; - python) - exceptions=( -path "./Examples/*" -o -path "./Fixtures/*" -o -path "./IntegrationTests/*" \ - -o -path "./Bundles/*" ) - matching_files=( -name '*.py' ) - cat > "$tmp" <<"EOF" -#!/usr/bin/env python3 -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift open source project -## -## Copyright (c) YEARS Apple Inc. and the Swift project authors -## Licensed under Apache License v2.0 with Runtime Library Exception -## -## See https://swift.org/LICENSE.txt for license information -## See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -## -##===----------------------------------------------------------------------===## -EOF - ;; - *) - echo >&2 "ERROR: unknown language '$language'" - ;; - esac - - expected_lines=$(cat "$tmp" | wc -l) - expected_sha=$(cat "$tmp" | shasum) - - ( - cd "$here/.." - { - find . \ - \( \! -path './.build/*' -a \ - \( "${matching_files[@]}" \) -a \ - \( \! \( "${exceptions[@]}" \) \) \) - - } | while read line; do - if [[ "$(cat "$line" | replace_acceptable_years | head -n $expected_lines | shasum)" != "$expected_sha" ]]; then - printf "\033[0;31mmissing headers in file '$line'!\033[0m\n" - diff -u <(cat "$line" | replace_acceptable_years | head -n $expected_lines) "$tmp" - exit 1 - fi - done - printf "\033[0;32mokay.\033[0m\n" - ) -done - -rm "$tmp"