From b9d611f90bbdbbc8eb116271175ae717a20a3485 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Mon, 3 Mar 2025 07:50:53 -0500 Subject: [PATCH 01/17] Fix support for building with `--static-swift-stdlib` for armv7 (#194) The [atomic library is used in the Swift stdlib](https://github.com/swiftlang/swift/blob/69756635819114a851023aa2e8e6fd87f51d094d/cmake/modules/AddSwift.cmake#L336-L338) for 32-bit architectures. As such, when attempting to compile a binary containing the static Swift stdlib using a Swift SDK for armv7, `-latomic` must be linked. This is not needed for regular cross-compilation, just when passing `--static-swift-stdlib`. I've added `-latomic` for "armv7" and added a test case in `LinuxRecipeTests` to ensure that it is included in the toolset.json file. Now, I can build my binary that contains the statically linked Swift stdlib with this command: ``` swift build --swift-sdk 6.0.3-RELEASE_debian_bookworm_armv7 --static-swift-stdlib ``` This generates a rather large binary when also linking other things such as Foundation and Dispatch, as is to be expected: ``` $ pwd ~/tmp/hummingbird-examples/hello $ du -hs .build/debug/App 123M .build/debug/App ``` But that's fine, because it works perfectly on the target (Raspberry Pi 2) and provides yet another option for compiling binaries using the Swift SDK. --- .../SwiftSDKRecipes/LinuxRecipe.swift | 5 +++++ .../SwiftSDKRecipes/LinuxRecipeTests.swift | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 9af0afc..c215b37 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -128,6 +128,11 @@ public struct LinuxRecipe: SwiftSDKRecipe { } else { swiftCompilerOptions.append("-use-ld=lld") + // 32-bit architectures require libatomic + if let arch = targetTriple.arch, arch.is32Bit { + swiftCompilerOptions.append("-latomic") + } + if self.hostSwiftSource != .preinstalled { toolset.linker = Toolset.ToolProperties(path: "ld.lld") } diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 436b705..3c21f0d 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -45,6 +45,7 @@ final class LinuxRecipeTests: XCTestCase { let testCases = [ ( swiftVersion: "5.9.2", + targetTriple: Triple("x86_64-unknown-linux-gnu"), expectedSwiftCompilerOptions: [ "-Xlinker", "-R/usr/lib/swift/linux/", "-Xclang-linker", "--ld-path=ld.lld" @@ -53,11 +54,22 @@ final class LinuxRecipeTests: XCTestCase { ), ( swiftVersion: "6.0.2", + targetTriple: Triple("aarch64-unknown-linux-gnu"), expectedSwiftCompilerOptions: [ "-Xlinker", "-R/usr/lib/swift/linux/", "-use-ld=lld" ], expectedLinkerPath: "ld.lld" + ), + ( + swiftVersion: "6.0.3", + targetTriple: Triple("armv7-unknown-linux-gnueabihf"), + expectedSwiftCompilerOptions: [ + "-Xlinker", "-R/usr/lib/swift/linux/", + "-use-ld=lld", + "-latomic" + ], + expectedLinkerPath: "ld.lld" ) ] @@ -65,7 +77,7 @@ final class LinuxRecipeTests: XCTestCase { let recipe = try self.createRecipe(swiftVersion: testCase.swiftVersion) var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: Triple("aarch64-unknown-linux-gnu") + toolset: &toolset, targetTriple: testCase.targetTriple ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, testCase.expectedSwiftCompilerOptions) XCTAssertEqual(toolset.linker?.path, testCase.expectedLinkerPath) From df46d2fd7cb29fd5354bfb4a78d1d39cbc8c991f Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Mon, 3 Mar 2025 07:59:53 -0500 Subject: [PATCH 02/17] Fix missing `lib/` directory for Ubuntu Noble, remove unused components (#193) Apparently I didn't test very well from PR #188, because when I went to use the Ubuntu Noble Swift SDK, I got this error: ``` error: link command failed with exit code 1 (use -v to see invocation) ld.lld: error: ~/.swiftpm/swift-sdks/6.0.3-RELEASE_ubuntu_noble_aarch64.artifactbundle/6.0.3-RELEASE_ubuntu_noble_aarch64/aarch64-unknown-linux-gnu/ubuntu-noble.sdk/usr/lib/aarch64-linux-gnu/libm.so:4: cannot find /lib/aarch64-linux-gnu/libm.so.6 inside ~/.swiftpm/swift-sdks/6.0.3-RELEASE_ubuntu_noble_aarch64/aarch64-unknown-linux-gnu/ubuntu-noble.sdk >>> GROUP ( /lib/aarch64-linux-gnu/libm.so.6 AS_NEEDED ( /lib/aarch64-linux-gnu/libmvec.so.1 ) ) >>> ^ clang: error: linker command failed with exit code 1 (use -v to see invocation) ld.lld: error: ~/.swiftpm/swift-sdks/6.0.3-RELEASE_ubuntu_noble_aarch64.artifactbundle/6.0.3-RELEASE_ubuntu_noble_aarch64/aarch64-unknown-linux-gnu/ubuntu-noble.sdk/usr/lib/aarch64-linux-gnu/libm.so:4: cannot find /lib/aarch64-linux-gnu/libm.so.6 inside /~/.swiftpm/swift-sdks/6.0.3-RELEASE_ubuntu_noble_aarch64.artifactbundle/6.0.3-RELEASE_ubuntu_noble_aarch64/aarch64-unknown-linux-gnu/ubuntu-noble.sdk >>> GROUP ( /lib/aarch64-linux-gnu/libm.so.6 AS_NEEDED ( /lib/aarch64-linux-gnu/libmvec.so.1 ) ) >>> ^ ``` Turns out, the packages for Ubuntu Noble do not come with a lib/ symlink included, so all I needed was to add a custom step to create that symlink from `ubuntu-noble.sdk/lib` -> `ubuntu-noble.sdk/usr/lib` and all is well. As a part of these changes, I also added a missing cleanup of the target toolchain to remove unused parts of the LinuxRecipe as is done for the [WebAssemblyRecipe](https://github.com/swiftlang/swift-sdk-generator/blob/a0ff972af294243ef9211534f7c9ba6f83672b82/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift#L120-L125). Doing this reduces the size of `ubuntu-noble.sdk` from 1GB to 721MB, when building the Swift SDK without docker. This can help for the end-to-end tests, and is nice to have for smaller Swift SDK distributions. Finally, I updated the EndToEndTests to provide `linuxDistributionVersion` and set the Swift60_Ubuntu tests to use "24.04" as a way to test this lib/ directory fix. This has me thinking we may want to think about end-to-end tests for each version of Ubuntu, like "20.04", "22.04", and "24.04", for each version of Swift. --- .../Generator/SwiftSDKGenerator+Fixup.swift | 8 ++++++++ .../SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift | 9 +++++++++ Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 9 ++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift index ea587cb..94df821 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift @@ -16,6 +16,14 @@ import SystemPackage import struct Foundation.Data extension SwiftSDKGenerator { + func createLibSymlink(sdkDirPath: FilePath) throws { + let libPath = sdkDirPath.appending("lib") + if !doesFileExist(at: libPath) { + logger.info("Adding lib symlink to usr/lib...") + try createSymlink(at: libPath, pointingTo: "usr/lib") + } + } + func fixAbsoluteSymlinks(sdkDirPath: FilePath) throws { logger.info("Fixing up absolute symlinks...") diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index c215b37..66a4dfe 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -294,6 +294,15 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) } + logger.info("Removing unused toolchain components from target SDK...") + try await generator.removeToolchainComponents( + sdkDirPath, + platforms: unusedTargetPlatforms, + libraries: unusedHostLibraries, + binaries: unusedHostBinaries + ) + + try await generator.createLibSymlink(sdkDirPath: sdkDirPath) try await generator.fixAbsoluteSymlinks(sdkDirPath: sdkDirPath) // Swift 6.1 and later do not throw warnings about the SDKSettings.json file missing, diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index ffbfc26..15b19c9 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -151,10 +151,11 @@ final class RepeatedBuildTests: XCTestCase { struct SDKConfiguration { var swiftVersion: String var linuxDistributionName: String + var linuxDistributionVersion: String var architecture: String var withDocker: Bool - var bundleName: String { "\(linuxDistributionName)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" } + var bundleName: String { "\(linuxDistributionName)_\(linuxDistributionVersion)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" } func withDocker(_ enabled: Bool = true) -> SDKConfiguration { var res = self @@ -308,6 +309,7 @@ final class Swift59_UbuntuEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", linuxDistributionName: "ubuntu", + linuxDistributionVersion: "22.04", architecture: "aarch64", withDocker: false ) @@ -337,6 +339,7 @@ final class Swift510_UbuntuEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.10.1", linuxDistributionName: "ubuntu", + linuxDistributionVersion: "22.04", architecture: "aarch64", withDocker: false ) @@ -366,6 +369,7 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "6.0.3", linuxDistributionName: "ubuntu", + linuxDistributionVersion: "24.04", architecture: "aarch64", withDocker: false ) @@ -395,6 +399,7 @@ final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", architecture: "aarch64", withDocker: true // RHEL-based SDKs can only be built from containers ) @@ -414,6 +419,7 @@ final class Swift510_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.10.1", linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", architecture: "aarch64", withDocker: true // RHEL-based SDKs can only be built from containers ) @@ -433,6 +439,7 @@ final class Swift60_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "6.0.3", linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", architecture: "aarch64", withDocker: true // RHEL-based SDKs can only be built from containers ) From 5fee5086f98d179302d02c8546274c245a5ce94b Mon Sep 17 00:00:00 2001 From: Mishal Shah Date: Mon, 10 Mar 2025 04:25:46 -0700 Subject: [PATCH 03/17] CI: Add support for GitHub Actions (#199) * [CI] Add support for GitHub Actions * Add locales locales-all libsqlite3-dev to the GitHub Actions * workflows: Disable tests; concentrate on fixing errors in soundness * workflows: Remove soundness.sh, which now comes from github-workflows Ironically, Utilities/soundness.sh fails the shellcheck soundness check. * workflows: Fix yamllint formatting * workflows: Exempt vendored code from swift format checks * workflows: Exempt ProcessExecutor and its tests from unacceptable language check "kill" in these files refers to a Unix command and signal and cannot be avoided. * Update .github/workflows/pull_request.yml * Unify license headers * Update `license_header_check_project_name` in workflow * Use `.license_header_template` * Fix most of the license checks * Update .licenseignore * Fix `.licenseignore` * workflows: Fix multiline string formatting for swift-format * workflows: Allow swift-format to fix whitespace * workflows: Snapshot default swift-format settings The .swift-format settings file was generated by: swift format dump-configuration > .swift-format We will need to configure or turn off some checks; this allows us to keep track of what settings we have changed. * workflows: Disable invasive swift-format checks swift format complains about some long-standing choices in this repository. For now we will disable these specific complaints. * workflows: Remove soundness check from CI compose files --------- Co-authored-by: Euan Harris Co-authored-by: Max Desiatov --- .github/workflows/pull_request.yml | 17 ++ .license_header_template | 12 + .licenseignore | 22 ++ .swift-format | 70 ++++++ .swiftformatignore | 2 + .unacceptablelanguageignore | 2 + Docker/docker-compose.yaml | 4 - Package.swift | 16 +- Sources/AsyncProcess/ChunkSequence.swift | 10 +- Sources/AsyncProcess/FileContentStream.swift | 16 +- .../ProcessExecutor+Convenience.swift | 43 ++-- Sources/AsyncProcess/ProcessExecutor.swift | 61 +++--- Sources/GeneratorCLI/GeneratorCLI.swift | 114 +++++----- Sources/Helpers/ThrowingDefer.swift | 4 +- Sources/Helpers/Vendor/Triple.swift | 4 +- .../_AsyncFileSystem/ConcurrencySupport.swift | 5 +- .../_AsyncFileSystem/WritableStream.swift | 20 +- .../Artifacts/DownloadableArtifacts.swift | 34 +-- .../Extensions/String+hasAnyPrefix.swift | 26 ++- .../Generator/SwiftSDKGenerator+Copy.swift | 48 ++-- .../SwiftSDKGenerator+Download.swift | 83 ++++--- .../SwiftSDKGenerator+Entrypoint.swift | 25 ++- .../Generator/SwiftSDKGenerator+Fixup.swift | 3 +- .../SwiftSDKGenerator+Metadata.swift | 21 +- .../Generator/SwiftSDKGenerator+Unpack.swift | 38 ++-- .../Generator/SwiftSDKGenerator.swift | 42 ++-- .../PathsConfiguration.swift | 3 +- .../PlatformModels/LinuxDistribution.swift | 16 +- .../PlatformModels/Triple.swift | 6 +- .../VersionsConfiguration.swift | 18 +- .../Queries/CMakeBuildQuery.swift | 1 + .../Queries/DownloadArtifactQuery.swift | 28 ++- .../Queries/DownloadFileQuery.swift | 3 +- .../Queries/TarExtractQuery.swift | 2 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 25 ++- .../SwiftSDKRecipes/SwiftSDKRecipe.swift | 13 +- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 61 ++++-- .../SystemUtils/ByteBuffer+Utils.swift | 4 +- .../SystemUtils/GeneratorError.swift | 19 +- .../SystemUtils/HTTPClient+Download.swift | 189 ++++++++-------- .../SwiftSDKGenerator/SystemUtils/Shell.swift | 5 +- .../SystemUtils/UnixName.swift | 16 +- .../SwiftSDKGenerator/SystemUtils/which.swift | 5 +- .../AsyncByteBufferLineSequenceTests.swift | 11 +- .../AsyncProcessTests/IntegrationTests.swift | 206 +++++++++--------- Tests/GeneratorEngineTests/EngineTests.swift | 24 +- Tests/HelpersTests/ThrowingDeferTests.swift | 38 ++-- Tests/MacrosTests/MacrosTests.swift | 52 ++--- .../ArchitectureMappingTests.swift | 43 ++-- .../EndToEndTests.swift | 79 ++++--- .../SwiftSDKGenerator+MetadataTests.swift | 2 +- .../SwiftSDKRecipes/LinuxRecipeTests.swift | 137 +++++++----- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 10 +- Utilities/soundness.sh | 150 ------------- 54 files changed, 1056 insertions(+), 852 deletions(-) create mode 100644 .github/workflows/pull_request.yml create mode 100644 .license_header_template create mode 100644 .licenseignore create mode 100644 .swift-format create mode 100644 .swiftformatignore create mode 100644 .unacceptablelanguageignore delete mode 100755 Utilities/soundness.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..c06cc36 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,17 @@ +name: Pull request + +on: + pull_request: + types: [opened, reopened, synchronize] + +jobs: + # tests: + # name: Test + # uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main + # with: + # linux_pre_build_command: apt-get update && apt-get install -y locales locales-all libsqlite3-dev + soundness: + name: Soundness + uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main + with: + api_breakage_check_enabled: false diff --git a/.license_header_template b/.license_header_template new file mode 100644 index 0000000..3f564ac --- /dev/null +++ b/.license_header_template @@ -0,0 +1,12 @@ +@@===----------------------------------------------------------------------===@@ +@@ +@@ This source file is part of the Swift open source project +@@ +@@ Copyright (c) YEARS Apple Inc. and the Swift project authors +@@ Licensed under Apache License v2.0 with Runtime Library Exception +@@ +@@ See https://swift.org/LICENSE.txt for license information +@@ See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +@@ +@@===----------------------------------------------------------------------===@@ + diff --git a/.licenseignore b/.licenseignore new file mode 100644 index 0000000..1957e72 --- /dev/null +++ b/.licenseignore @@ -0,0 +1,22 @@ +.gitignore +.swiftformat +.swiftformatignore +.editorconfig +.unacceptablelanguageignore +Brewfile +Package.swift +Package.resolved +*.md +*.txt +*.yml +**/.editorconfig +**/*.docc/** +**/*.entitlements +**/*.input +**/*.modulemap +**/*.plist +**/*.xcodeproj/** +**/CODEOWNERS +**/Dockerfile +**/Package.swift +Utilities/git.commit.template diff --git a/.swift-format b/.swift-format new file mode 100644 index 0000000..e345c2e --- /dev/null +++ b/.swift-format @@ -0,0 +1,70 @@ +{ + "fileScopedDeclarationPrivacy" : { + "accessLevel" : "private" + }, + "indentConditionalCompilationBlocks" : true, + "indentSwitchCaseLabels" : false, + "indentation" : { + "spaces" : 2 + }, + "lineBreakAroundMultilineExpressionChainComponents" : false, + "lineBreakBeforeControlFlowKeywords" : false, + "lineBreakBeforeEachArgument" : false, + "lineBreakBeforeEachGenericRequirement" : false, + "lineLength" : 100, + "maximumBlankLines" : 1, + "multiElementCollectionTrailingCommas" : true, + "noAssignmentInExpressions" : { + "allowedFunctions" : [ + "XCTAssertNoThrow" + ] + }, + "prioritizeKeepingFunctionOutputTogether" : false, + "respectsExistingLineBreaks" : true, + "rules" : { + "AllPublicDeclarationsHaveDocumentation" : false, + "AlwaysUseLiteralForEmptyCollectionInit" : false, + "AlwaysUseLowerCamelCase" : false, + "AmbiguousTrailingClosureOverload" : true, + "BeginDocumentationCommentWithOneLineSummary" : false, + "DoNotUseSemicolons" : true, + "DontRepeatTypeInStaticProperties" : true, + "FileScopedDeclarationPrivacy" : true, + "FullyIndirectEnum" : true, + "GroupNumericLiterals" : true, + "IdentifiersMustBeASCII" : true, + "NeverForceUnwrap" : false, + "NeverUseForceTry" : false, + "NeverUseImplicitlyUnwrappedOptionals" : false, + "NoAccessLevelOnExtensionDeclaration" : true, + "NoAssignmentInExpressions" : true, + "NoBlockComments" : true, + "NoCasesWithOnlyFallthrough" : true, + "NoEmptyTrailingClosureParentheses" : true, + "NoLabelsInCasePatterns" : true, + "NoLeadingUnderscores" : false, + "NoParensAroundConditions" : true, + "NoPlaygroundLiterals" : true, + "NoVoidReturnOnFunctionSignature" : true, + "OmitExplicitReturns" : false, + "OneCasePerLine" : true, + "OneVariableDeclarationPerLine" : true, + "OnlyOneTrailingClosureArgument" : true, + "OrderedImports" : true, + "ReplaceForEachWithForLoop" : true, + "ReturnVoidInsteadOfEmptyTuple" : true, + "TypeNamesShouldBeCapitalized" : true, + "UseEarlyExits" : false, + "UseExplicitNilCheckInConditions" : true, + "UseLetInEveryBoundCaseVariable" : false, + "UseShorthandTypeNames" : true, + "UseSingleLinePropertyGetter" : true, + "UseSynthesizedInitializer" : true, + "UseTripleSlashForDocumentationComments" : true, + "UseWhereClausesInForLoops" : false, + "ValidateDocumentationComments" : false + }, + "spacesAroundRangeFormationOperators" : false, + "tabWidth" : 8, + "version" : 1 +} diff --git a/.swiftformatignore b/.swiftformatignore new file mode 100644 index 0000000..ebdee05 --- /dev/null +++ b/.swiftformatignore @@ -0,0 +1,2 @@ +Sources/Helpers/Vendor/* +Sources/AsyncProcess/ProcessExecutor.swift diff --git a/.unacceptablelanguageignore b/.unacceptablelanguageignore new file mode 100644 index 0000000..24ab69f --- /dev/null +++ b/.unacceptablelanguageignore @@ -0,0 +1,2 @@ +Sources/AsyncProcess/ProcessExecutor.swift +Tests/AsyncProcessTests/IntegrationTests.swift diff --git a/Docker/docker-compose.yaml b/Docker/docker-compose.yaml index 3db9408..95378ce 100644 --- a/Docker/docker-compose.yaml +++ b/Docker/docker-compose.yaml @@ -19,10 +19,6 @@ services: - ..:/code:z working_dir: /code - soundness: - <<: *common - command: /bin/bash -xcl "swift -version && uname -a && ./Utilities/soundness.sh" - test: <<: *common environment: diff --git a/Package.swift b/Package.swift index 8094ef8..832b742 100644 --- a/Package.swift +++ b/Package.swift @@ -11,7 +11,7 @@ let package = Package( .executable( name: "swift-sdk-generator", targets: ["GeneratorCLI"] - ), + ) ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. @@ -23,7 +23,7 @@ let package = Package( .product(name: "ArgumentParser", package: "swift-argument-parser"), ], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), + .enableExperimentalFeature("StrictConcurrency=complete") ] ), .target( @@ -38,22 +38,22 @@ let package = Package( ], exclude: ["Dockerfiles"], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), + .enableExperimentalFeature("StrictConcurrency=complete") ] ), .testTarget( name: "SwiftSDKGeneratorTests", dependencies: [ - "SwiftSDKGenerator", + "SwiftSDKGenerator" ], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), + .enableExperimentalFeature("StrictConcurrency=complete") ] ), .testTarget( name: "GeneratorEngineTests", dependencies: [ - "Helpers", + "Helpers" ] ), .target( @@ -66,13 +66,13 @@ let package = Package( ], exclude: ["Vendor/README.md"], swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency=complete"), + .enableExperimentalFeature("StrictConcurrency=complete") ] ), .testTarget( name: "HelpersTests", dependencies: [ - "Helpers", + "Helpers" ] ), .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"), diff --git a/Sources/AsyncProcess/ChunkSequence.swift b/Sources/AsyncProcess/ChunkSequence.swift index 8daa1a3..fe60529 100644 --- a/Sources/AsyncProcess/ChunkSequence.swift +++ b/Sources/AsyncProcess/ChunkSequence.swift @@ -13,9 +13,9 @@ import NIO #if os(Linux) || os(Android) || os(Windows) -@preconcurrency import Foundation + @preconcurrency import Foundation #else -import Foundation + import Foundation #endif public struct IllegalStreamConsumptionError: Error { @@ -53,9 +53,9 @@ public struct ChunkSequence: AsyncSequence & Sendable { } else { throw IllegalStreamConsumptionError( description: """ - Either `.discard`ed, `.inherit`ed or redirected this stream to a `.fileHandle`, - cannot also consume it. To consume, please `.stream` it. - """ + Either `.discard`ed, `.inherit`ed or redirected this stream to a `.fileHandle`, + cannot also consume it. To consume, please `.stream` it. + """ ) } } diff --git a/Sources/AsyncProcess/FileContentStream.swift b/Sources/AsyncProcess/FileContentStream.swift index c704ea9..c0ee126 100644 --- a/Sources/AsyncProcess/FileContentStream.swift +++ b/Sources/AsyncProcess/FileContentStream.swift @@ -255,7 +255,8 @@ private final class ReadIntoAsyncChannelHandler: ChannelDuplexHandler { extension FileHandle { func fileContentStream(eventLoop: EventLoop) throws -> FileContentStream { - let asyncBytes = try FileContentStream(fileDescriptor: self.fileDescriptor, eventLoop: eventLoop) + let asyncBytes = try FileContentStream( + fileDescriptor: self.fileDescriptor, eventLoop: eventLoop) try self.close() return asyncBytes } @@ -272,8 +273,8 @@ extension FileContentStream { } } -public extension AsyncSequence where Element == ByteBuffer, Self: Sendable { - func splitIntoLines( +extension AsyncSequence where Element == ByteBuffer, Self: Sendable { + public func splitIntoLines( dropTerminator: Bool = true, maximumAllowableBufferSize: Int = 1024 * 1024, dropLastChunkIfNoNewline: Bool = false @@ -286,14 +287,13 @@ public extension AsyncSequence where Element == ByteBuffer, Self: Sendable { ) } - var strings: AsyncMapSequence { + public var strings: AsyncMapSequence { self.map { String(buffer: $0) } } } public struct AsyncByteBufferLineSequence: AsyncSequence & Sendable - where Base: AsyncSequence, Base.Element == ByteBuffer -{ +where Base: AsyncSequence, Base.Element == ByteBuffer { public typealias Element = ByteBuffer private let underlying: Base private let dropTerminator: Bool @@ -329,7 +329,9 @@ public struct AsyncByteBufferLineSequence: AsyncSequence & Senda self.buffer.last?.readableBytesView } - mutating func concatenateEverything(upToLastChunkLengthToConsume lastLength: Int) -> ByteBuffer { + mutating func concatenateEverything(upToLastChunkLengthToConsume lastLength: Int) + -> ByteBuffer + { var output = ByteBuffer() output.reserveCapacity(lastLength + self.byteCountButLast) diff --git a/Sources/AsyncProcess/ProcessExecutor+Convenience.swift b/Sources/AsyncProcess/ProcessExecutor+Convenience.swift index b7815cb..00c6607 100644 --- a/Sources/AsyncProcess/ProcessExecutor+Convenience.swift +++ b/Sources/AsyncProcess/ProcessExecutor+Convenience.swift @@ -53,7 +53,7 @@ public struct OutputLoggingSettings: Sendable { } } -public extension ProcessExecutor { +extension ProcessExecutor { /// Run child process, discarding all its output. /// /// - note: The `environment` defaults to the empty environment. @@ -69,7 +69,7 @@ public extension ProcessExecutor { /// don't want to /// provide input. /// - logger: Where to log diagnostic messages to (default to no where) - static func run( + public static func run( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -106,7 +106,7 @@ public extension ProcessExecutor { /// provide input. /// - logger: Where to log diagnostic and output messages to /// - logConfiguration: How to log the output lines - static func runLogOutput( + public static func runLogOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -128,7 +128,9 @@ public extension ProcessExecutor { return try await withThrowingTaskGroup(of: ProcessExitReason?.self) { group in group.addTask { for try await (stream, line) in await merge( - exe.standardOutput.splitIntoLines().strings.map { (ProcessOutputStream.standardOutput, $0) }, + exe.standardOutput.splitIntoLines().strings.map { + (ProcessOutputStream.standardOutput, $0) + }, exe.standardError.splitIntoLines().strings.map { (ProcessOutputStream.standardError, $0) } ) { logger.log( @@ -171,12 +173,12 @@ public extension ProcessExecutor { /// - splitOutputIntoLines: Whether to call the closure with full lines (`true`) or arbitrary chunks of output /// (`false`) /// - logger: Where to log diagnostic and output messages to - static func runProcessingOutput( + public static func runProcessingOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], standardInput: StandardInput, - outputProcessor: @escaping @Sendable (ProcessOutputStream, ByteBuffer) async throws -> (), + outputProcessor: @escaping @Sendable (ProcessOutputStream, ByteBuffer) async throws -> Void, splitOutputIntoLines: Bool = false, environment: [String: String] = [:], logger: Logger = ProcessExecutor.disableLogging @@ -225,11 +227,11 @@ public extension ProcessExecutor { } } - struct TooMuchProcessOutputError: Error, Sendable & Hashable { + public struct TooMuchProcessOutputError: Error, Sendable & Hashable { public var stream: ProcessOutputStream } - struct ProcessExitReasonAndOutput: Sendable & Hashable { + public struct ProcessExitReasonAndOutput: Sendable & Hashable { public var exitReason: ProcessExitReason public var standardOutput: ByteBuffer? public var standardError: ByteBuffer? @@ -260,7 +262,7 @@ public extension ProcessExecutor { /// - collectStandardError: If `true`, collect all of the child process' standard error into memory, discard if /// `false` /// - logger: Where to log diagnostic and output messages to - static func runCollectingOutput( + public static func runCollectingOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -287,7 +289,9 @@ public extension ProcessExecutor { if collectStandardOutput { var output: ByteBuffer? = nil for try await chunk in await exe.standardOutput { - guard (output?.readableBytes ?? 0) + chunk.readableBytes <= perStreamCollectionLimitBytes else { + guard + (output?.readableBytes ?? 0) + chunk.readableBytes <= perStreamCollectionLimitBytes + else { throw TooMuchProcessOutputError(stream: .standardOutput) } output.setOrWriteImmutableBuffer(chunk) @@ -302,7 +306,9 @@ public extension ProcessExecutor { if collectStandardError { var output: ByteBuffer? = nil for try await chunk in await exe.standardError { - guard (output?.readableBytes ?? 0) + chunk.readableBytes <= perStreamCollectionLimitBytes else { + guard + (output?.readableBytes ?? 0) + chunk.readableBytes <= perStreamCollectionLimitBytes + else { throw TooMuchProcessOutputError(stream: .standardError) } output.setOrWriteImmutableBuffer(chunk) @@ -317,7 +323,8 @@ public extension ProcessExecutor { try await .exitReason(exe.run()) } - var allInfo = ProcessExitReasonAndOutput(exitReason: .exit(-1), standardOutput: nil, standardError: nil) + var allInfo = ProcessExitReasonAndOutput( + exitReason: .exit(-1), standardOutput: nil, standardError: nil) while let next = try await group.next() { switch next { case let .exitReason(exitReason): @@ -333,7 +340,7 @@ public extension ProcessExecutor { } } -public extension ProcessExecutor { +extension ProcessExecutor { /// Run child process, discarding all its output. /// /// - note: The `environment` defaults to the empty environment. @@ -346,7 +353,7 @@ public extension ProcessExecutor { /// If you want to inherit the calling process' environment into the child, specify /// `ProcessInfo.processInfo.environment` /// - logger: Where to log diagnostic messages to (default to no where) - static func run( + public static func run( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -376,7 +383,7 @@ public extension ProcessExecutor { /// `ProcessInfo.processInfo.environment` /// - logger: Where to log diagnostic and output messages to /// - logConfiguration: How to log the output lines - static func runLogOutput( + public static func runLogOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], @@ -410,11 +417,11 @@ public extension ProcessExecutor { /// - splitOutputIntoLines: Whether to call the closure with full lines (`true`) or arbitrary chunks of output /// (`false`) /// - logger: Where to log diagnostic and output messages to - static func runProcessingOutput( + public static func runProcessingOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], - outputProcessor: @escaping @Sendable (ProcessOutputStream, ByteBuffer) async throws -> (), + outputProcessor: @escaping @Sendable (ProcessOutputStream, ByteBuffer) async throws -> Void, splitOutputIntoLines: Bool = false, environment: [String: String] = [:], logger: Logger = ProcessExecutor.disableLogging @@ -447,7 +454,7 @@ public extension ProcessExecutor { /// - collectStandardError: If `true`, collect all of the child process' standard error into memory, discard if /// `false` /// - logger: Where to log diagnostic and output messages to - static func runCollectingOutput( + public static func runCollectingOutput( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], diff --git a/Sources/AsyncProcess/ProcessExecutor.swift b/Sources/AsyncProcess/ProcessExecutor.swift index 59f1d0a..80429a6 100644 --- a/Sources/AsyncProcess/ProcessExecutor.swift +++ b/Sources/AsyncProcess/ProcessExecutor.swift @@ -11,13 +11,12 @@ //===----------------------------------------------------------------------===// import Atomics +import Foundation import Logging import NIO @_exported import struct SystemPackage.FileDescriptor -import Foundation - public struct ProcessOutputStream: Sendable & Hashable & CustomStringConvertible { enum Backing { case standardOutput @@ -159,7 +158,8 @@ public final actor ProcessExecutor { private let standardErrorWriteHandle: FileHandle? private let _standardOutput: ChunkSequence private let _standardError: ChunkSequence - private let processIsRunningApproximation = ManagedAtomic(RunningStateApproximation.neverStarted.rawValue) + private let processIsRunningApproximation = ManagedAtomic( + RunningStateApproximation.neverStarted.rawValue) private let processOutputConsumptionApproximation = ManagedAtomic(UInt8(0)) private let processPid = ManagedAtomic(pid_t(0)) private let ownsStandardOutputWriteHandle: Bool @@ -388,14 +388,14 @@ public final actor ProcessExecutor { { () -> Bool in guard outputConsumptionState.contains([.stdoutConsumed]) - || outputConsumptionState.contains([.stdoutNotStreamed]) + || outputConsumptionState.contains([.stdoutNotStreamed]) else { return false } guard outputConsumptionState.contains([.stderrConsumed]) - || outputConsumptionState.contains([.stderrNotStreamed]) + || outputConsumptionState.contains([.stderrNotStreamed]) else { return false } @@ -459,7 +459,7 @@ public final actor ProcessExecutor { case .processHasExited, .killedTheProcess: break loop case .processStillAlive: - () // gotta continue + () // gotta continue } } else { logger.debug("child process already dead") @@ -482,13 +482,13 @@ public final actor ProcessExecutor { public func run() async throws -> ProcessExitReason { let p = Process() #if canImport(Darwin) - if #available(macOS 13.0, *) { - p.executableURL = URL(filePath: self.executable) - } else { - p.launchPath = self.executable - } + if #available(macOS 13.0, *) { + p.executableURL = URL(filePath: self.executable) + } else { + p.launchPath = self.executable + } #else - p.executableURL = URL(fileURLWithPath: self.executable) + p.executableURL = URL(fileURLWithPath: self.executable) #endif p.arguments = self.arguments p.environment = self.environment @@ -559,15 +559,16 @@ public final actor ProcessExecutor { desired: RunningStateApproximation.finishedExecuting.rawValue, ordering: .relaxed ) - terminationStreamProducer.finish() // The termination handler will never have fired. - assert(worked) // We just set it to running above, shouldn't be able to race (no `await`). - assert(original == RunningStateApproximation.running.rawValue) // We compare-and-exchange it. + terminationStreamProducer.finish() // The termination handler will never have fired. + assert(worked) // We just set it to running above, shouldn't be able to race (no `await`). + assert(original == RunningStateApproximation.running.rawValue) // We compare-and-exchange it. throw error } // At this point, the process is running, we should therefore have a process ID (unless we're already dead). let childPid = p.processIdentifier - _ = self.processPid.compareExchange(expected: 0, desired: childPid, ordering: .sequentiallyConsistent) + _ = self.processPid.compareExchange( + expected: 0, desired: childPid, ordering: .sequentiallyConsistent) assert(childPid != 0 || !p.isRunning) self.logger.debug( "running command", @@ -578,12 +579,12 @@ public final actor ProcessExecutor { ] ) - try! self.standardInputPipe?.fileHandleForReading.close() // Must work. + try! self.standardInputPipe?.fileHandleForReading.close() // Must work. if self.ownsStandardOutputWriteHandle { - try! self.standardOutputWriteHandle?.close() // Must work. + try! self.standardOutputWriteHandle?.close() // Must work. } if self.ownsStandardErrorWriteHandle { - try! self.standardErrorWriteHandle?.close() // Must work. + try! self.standardErrorWriteHandle?.close() // Must work. } @Sendable @@ -611,9 +612,11 @@ public final actor ProcessExecutor { await withTaskGroup(of: Void.self) { triggerTeardownGroup in triggerTeardownGroup.addTask { // wait until cancelled - do { while true { - try await Task.sleep(nanoseconds: 1_000_000_000) - } } catch {} + do { + while true { + try await Task.sleep(nanoseconds: 1_000_000_000) + } + } catch {} let isRunning = self.processIsRunningApproximation.load(ordering: .relaxed) guard isRunning != RunningStateApproximation.finishedExecuting.rawValue else { @@ -639,7 +642,7 @@ public final actor ProcessExecutor { } let result = await waitForChildToExit() - triggerTeardownGroup.cancelAll() // This triggers the teardown + triggerTeardownGroup.cancelAll() // This triggers the teardown return result } } @@ -663,7 +666,7 @@ public final actor ProcessExecutor { exitReason = result } } - return exitReason! // must work because the real task will return a reason (or throw) + return exitReason! // must work because the real task will return a reason (or throw) } } @@ -679,21 +682,21 @@ public final actor ProcessExecutor { } } -public extension ProcessExecutor { +extension ProcessExecutor { /// A globally shared, singleton `EventLoopGroup` that's suitable for ``ProcessExecutor``. /// /// At present this is always `MultiThreadedEventLoopGroup.singleton`. - static var defaultEventLoopGroup: any EventLoopGroup { + public static var defaultEventLoopGroup: any EventLoopGroup { globalDefaultEventLoopGroup } /// The default `Logger` for ``ProcessExecutor`` that's used if you do not override it. It won't log anything. - static var disableLogging: Logger { + public static var disableLogging: Logger { globalDisableLoggingLogger } } -public extension ProcessExecutor { +extension ProcessExecutor { /// Create a ``ProcessExecutor`` to spawn a single child process. /// /// - note: The `environment` defaults to the empty environment. @@ -712,7 +715,7 @@ public extension ProcessExecutor { /// ``ProcessOutput/stream`` /// which requires to consume it via ``ProcessExecutor/standardError``. /// - logger: Where to log diagnostic messages to (default to no where) - init( + public init( group: EventLoopGroup = ProcessExecutor.defaultEventLoopGroup, executable: String, _ arguments: [String], diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index a322f10..82667af 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -11,8 +11,10 @@ //===----------------------------------------------------------------------===// import ArgumentParser +import Foundation import Logging import SwiftSDKGenerator + import struct SystemPackage.FilePath @main @@ -54,10 +56,10 @@ struct GeneratorCLI: AsyncParsableCommand { } #if canImport(Darwin) - // On Darwin platforms Dispatch's signal source uses kqueue and EVFILT_SIGNAL for - // delivering signals. This exists alongside but with lower precedence than signal and - // sigaction: ignore signal handling here to kqueue can deliver signals. - signal(SIGINT, SIG_IGN) + // On Darwin platforms Dispatch's signal source uses kqueue and EVFILT_SIGNAL for + // delivering signals. This exists alongside but with lower precedence than signal and + // sigaction: ignore signal handling here to kqueue can deliver signals. + signal(SIGINT, SIG_IGN) #endif let signalSource = DispatchSource.makeSignalSource(signal: SIGINT) signalSource.setEventHandler { @@ -67,7 +69,9 @@ struct GeneratorCLI: AsyncParsableCommand { try await generatorTask.value } - logger.info("Generator run finished successfully.", metadata: ["elapsedTime": .string(elapsed.intervalString)]) + logger.info( + "Generator run finished successfully.", + metadata: ["elapsedTime": .string(elapsed.intervalString)]) } } @@ -85,14 +89,15 @@ extension GeneratorCLI { @Option( help: """ - Name of the SDK bundle. Defaults to a string composed of Swift version, Linux distribution, Linux release \ - and target CPU architecture. - """ + Name of the SDK bundle. Defaults to a string composed of Swift version, Linux distribution, Linux release \ + and target CPU architecture. + """ ) var sdkName: String? = nil @Flag( - help: "Experimental: avoid cleaning up toolchain and SDK directories and regenerate the SDK bundle incrementally." + help: + "Experimental: avoid cleaning up toolchain and SDK directories and regenerate the SDK bundle incrementally." ) var incremental: Bool = false @@ -101,9 +106,9 @@ extension GeneratorCLI { @Option( help: """ - Branch of Swift to use when downloading nightly snapshots. Specify `development` for snapshots off the `main` \ - branch of Swift open source project repositories. - """ + Branch of Swift to use when downloading nightly snapshots. Specify `development` for snapshots off the `main` \ + branch of Swift open source project repositories. + """ ) var swiftBranch: String? = nil @@ -112,40 +117,40 @@ extension GeneratorCLI { @Option( help: """ - Path to the Swift toolchain package containing the Swift compiler that runs on the host platform. - """ + Path to the Swift toolchain package containing the Swift compiler that runs on the host platform. + """ ) var hostSwiftPackagePath: String? = nil @Flag( inversion: .prefixedNo, help: """ - Whether or not to include the host toolchain in the Swift SDK. - If the host toolchain is not included, this makes the Swift SDK compatible with any host, \ - but requires exactly the same version of the swift.org toolchain to be installed for it to work. - """ + Whether or not to include the host toolchain in the Swift SDK. + If the host toolchain is not included, this makes the Swift SDK compatible with any host, \ + but requires exactly the same version of the swift.org toolchain to be installed for it to work. + """ ) var hostToolchain: Bool = false @Option( help: """ - Path to the Swift toolchain package containing the Swift standard library that runs on the target platform. - """ + Path to the Swift toolchain package containing the Swift standard library that runs on the target platform. + """ ) var targetSwiftPackagePath: String? = nil @Option( help: """ - The host triple of the bundle. Defaults to a triple of the machine this generator is \ - running on if unspecified. - """ + The host triple of the bundle. Defaults to a triple of the machine this generator is \ + running on if unspecified. + """ ) var host: Triple? = nil @Option( help: """ - The target triple of the bundle. The default depends on a recipe used for SDK generation. Pass `--help` to a specific recipe subcommand for more details. - """ + The target triple of the bundle. The default depends on a recipe used for SDK generation. Pass `--help` to a specific recipe subcommand for more details. + """ ) var target: Triple? = nil @@ -161,7 +166,8 @@ extension GeneratorCLI { let current = try SwiftSDKGenerator.getCurrentTriple(isVerbose: self.verbose) if let arch = hostArch { let target = Triple(arch: arch, vendor: current.vendor!, os: current.os!) - appLogger.warning("deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`") + appLogger.warning( + "deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`") return target } return current @@ -173,8 +179,8 @@ extension GeneratorCLI { commandName: "make-linux-sdk", abstract: "Generate a Swift SDK bundle for Linux.", discussion: """ - The default `--target` triple is Linux with the same CPU architecture with host triple - """ + The default `--target` triple is Linux with the same CPU architecture with host triple + """ ) @OptionGroup @@ -191,18 +197,18 @@ extension GeneratorCLI { @Option( help: """ - Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`. - """, + Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`. + """, transform: LinuxDistribution.Name.init(nameString:) ) var linuxDistributionName = LinuxDistribution.Name.ubuntu @Option( help: """ - Version of the Linux distribution used as a target platform. - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). - """ + Version of the Linux distribution used as a target platform. + Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. + Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). + """ ) var linuxDistributionVersion: String? @@ -212,7 +218,8 @@ extension GeneratorCLI { } if let arch = generatorOptions.targetArch { let target = Triple(arch: arch, vendor: nil, os: .linux, environment: .gnu) - appLogger.warning("deprecated: Please use `--target \(target.triple)` instead of `--target-arch \(arch)`") + appLogger.warning( + "deprecated: Please use `--target \(target.triple)` instead of `--target-arch \(arch)`") } return Triple(arch: hostTriple.arch!, vendor: nil, os: .linux, environment: .gnu) } @@ -230,8 +237,10 @@ extension GeneratorCLI { case .ubuntu: linuxDistributionDefaultVersion = "22.04" } - let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion - let linuxDistribution = try LinuxDistribution(name: linuxDistributionName, version: linuxDistributionVersion) + let linuxDistributionVersion = + self.linuxDistributionVersion ?? linuxDistributionDefaultVersion + let linuxDistribution = try LinuxDistribution( + name: linuxDistributionName, version: linuxDistributionVersion) let hostTriple = try self.generatorOptions.deriveHostTriple() let targetTriple = self.deriveTargetTriple(hostTriple: hostTriple) @@ -249,7 +258,8 @@ extension GeneratorCLI { includeHostToolchain: self.generatorOptions.hostToolchain, logger: loggerWithLevel(from: self.generatorOptions) ) - try await GeneratorCLI.run(recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions) + try await GeneratorCLI.run( + recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions) } func isInvokedAsDefaultSubcommand() -> Bool { @@ -272,8 +282,8 @@ extension GeneratorCLI { commandName: "make-wasm-sdk", abstract: "Experimental: Generate a Swift SDK bundle for WebAssembly.", discussion: """ - The default `--target` triple is wasm32-unknown-wasi - """ + The default `--target` triple is wasm32-unknown-wasi + """ ) @OptionGroup @@ -281,8 +291,8 @@ extension GeneratorCLI { @Option( help: """ - Path to the WASI sysroot directory containing the WASI libc headers and libraries. - """ + Path to the WASI sysroot directory containing the WASI libc headers and libraries. + """ ) var wasiSysroot: String @@ -305,32 +315,34 @@ extension GeneratorCLI { logger: loggerWithLevel(from: self.generatorOptions) ) let targetTriple = self.deriveTargetTriple() - try await GeneratorCLI.run(recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions) + try await GeneratorCLI.run( + recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions) } } } -import Foundation - extension Duration { var intervalString: String { let reference = Date() let date = Date(timeInterval: TimeInterval(self.components.seconds), since: reference) - let components = Calendar.current.dateComponents([.hour, .minute, .second], from: reference, to: date) + let components = Calendar.current.dateComponents( + [.hour, .minute, .second], from: reference, to: date) if let hours = components.hour, hours > 0 { #if !canImport(Darwin) && compiler(<6.0) - return String(format: "%02d:%02d:%02d", hours, components.minute ?? 0, components.second ?? 0) + return String( + format: "%02d:%02d:%02d", hours, components.minute ?? 0, components.second ?? 0) #else - return self.formatted() + return self.formatted() #endif } else if let minutes = components.minute, minutes > 0 { #if !canImport(Darwin) && compiler(<6.0) - let seconds = components.second ?? 0 - return "\(minutes) minute\(minutes != 1 ? "s" : "") \(seconds) second\(seconds != 1 ? "s" : "")" + let seconds = components.second ?? 0 + return + "\(minutes) minute\(minutes != 1 ? "s" : "") \(seconds) second\(seconds != 1 ? "s" : "")" #else - return "\(self.formatted(.time(pattern: .minuteSecond))) seconds" + return "\(self.formatted(.time(pattern: .minuteSecond))) seconds" #endif } else { return "\(components.second ?? 0) seconds" diff --git a/Sources/Helpers/ThrowingDefer.swift b/Sources/Helpers/ThrowingDefer.swift index 9fec320..899c581 100644 --- a/Sources/Helpers/ThrowingDefer.swift +++ b/Sources/Helpers/ThrowingDefer.swift @@ -22,7 +22,7 @@ /// - SeeAlso: ``withAsyncThrowing(do:defer:)`` public func withThrowing( do work: () throws -> T, - defer deferred: () throws -> () + defer deferred: () throws -> Void ) throws -> T { do { let result = try work() @@ -46,7 +46,7 @@ public func withThrowing( /// - SeeAlso: ``withThrowing(do:defer:)`` public func withAsyncThrowing( do work: @Sendable () async throws -> T, - defer deferred: @Sendable () async throws -> () + defer deferred: @Sendable () async throws -> Void ) async throws -> T { do { let result = try await work() diff --git a/Sources/Helpers/Vendor/Triple.swift b/Sources/Helpers/Vendor/Triple.swift index fd256d4..0fe731e 100644 --- a/Sources/Helpers/Vendor/Triple.swift +++ b/Sources/Helpers/Vendor/Triple.swift @@ -1,6 +1,6 @@ -//===--------------- Triple.swift - Swift Target Triples ------------------===// +//===----------------------------------------------------------------------===// // -// This source file is part of the Swift.org open source project +// This source file is part of the Swift open source project // // Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift index b62b190..50d3c54 100644 --- a/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift @@ -1,4 +1,3 @@ - //===----------------------------------------------------------------------===// // // This source file is part of the Swift open source project @@ -6,8 +5,8 @@ // Copyright (c) 2020-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // //===----------------------------------------------------------------------===// diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift index 844ba04..dd5657a 100644 --- a/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift @@ -1,12 +1,14 @@ -/* - This source file is part of the Swift.org open source project - - Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors - Licensed under Apache License v2.0 with Runtime Library Exception - - See http://swift.org/LICENSE.txt for license information - See http://swift.org/CONTRIBUTORS.txt for Swift project authors -*/ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// import _Concurrency diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index 88fb7d5..ec4725e 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -10,12 +10,13 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.URL import Helpers + +import struct Foundation.URL import struct SystemPackage.FilePath -private extension Triple { - var llvmBinaryURLSuffix: String { +extension Triple { + fileprivate var llvmBinaryURLSuffix: String { switch (self.os, self.arch) { case (.linux, .aarch64): return "aarch64-linux-gnu" case (.linux, .x86_64): return "x86_64-linux-gnu-ubuntu-22.04" @@ -53,7 +54,8 @@ struct DownloadableArtifacts: Sendable { if hostTriple.os == .linux { // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts - let linuxArchSuffix = hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + let linuxArchSuffix = + hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" self.hostSwift = .init( remoteURL: versions.swiftDownloadURL( subdirectory: "amazonlinux2\(linuxArchSuffix)", @@ -80,12 +82,12 @@ struct DownloadableArtifacts: Sendable { self.hostLLVM = .init( remoteURL: URL( string: """ - https://github.com/llvm/llvm-project/releases/download/llvmorg-\( - versions.lldVersion - )/clang+llvm-\( - versions.lldVersion - )-\(hostTriple.llvmBinaryURLSuffix).tar.xz - """ + https://github.com/llvm/llvm-project/releases/download/llvmorg-\( + versions.lldVersion + )/clang+llvm-\( + versions.lldVersion + )-\(hostTriple.llvmBinaryURLSuffix).tar.xz + """ )!, localPath: paths.artifactsCachePath .appending("host_llvm_\(versions.lldVersion)_\(hostTriple.triple).tar.xz"), @@ -104,12 +106,12 @@ struct DownloadableArtifacts: Sendable { self.hostLLVM = .init( remoteURL: URL( string: """ - https://github.com/llvm/llvm-project/releases/download/llvmorg-\( - self.versions.lldVersion - )/llvm-project-\( - self.versions.lldVersion - ).src.tar.xz - """ + https://github.com/llvm/llvm-project/releases/download/llvmorg-\( + self.versions.lldVersion + )/llvm-project-\( + self.versions.lldVersion + ).src.tar.xz + """ )!, localPath: self.paths.artifactsCachePath .appending("llvm_\(self.versions.lldVersion).src.tar.xz"), diff --git a/Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift b/Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift index 139e371..0ad671b 100644 --- a/Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift +++ b/Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift @@ -1,10 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + extension String { - func hasAnyPrefix(from array: [String]) -> Bool { - for item in array { - if self.hasPrefix(item) { - return true - } - } - return false + func hasAnyPrefix(from array: [String]) -> Bool { + for item in array { + if self.hasPrefix(item) { + return true + } } + return false + } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 5b9afae..e561cd2 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -import SystemPackage import Foundation +import SystemPackage extension SwiftSDKGenerator { func copyTargetSwiftFromDocker( @@ -34,23 +34,23 @@ extension SwiftSDKGenerator { try await generator.runOnDockerContainer( id: containerID, command: #""" - sh -c ' - chmod +w /usr/lib64 - cd /usr/lib64 - for n in *; do - destination=$(readlink $n) - echo $destination | grep "\.\." && \ - rm -f $n && \ - ln -s $(basename $destination) $n - done - rm -rf pm-utils - ' - """# + sh -c ' + chmod +w /usr/lib64 + cd /usr/lib64 + for n in *; do + destination=$(readlink $n) + echo $destination | grep "\.\." && \ + rm -f $n && \ + ln -s $(basename $destination) $n + done + rm -rf pm-utils + ' + """# ) } if case let containerLib64 = FilePath("/usr/lib64"), - try await generator.doesPathExist(containerLib64, inContainer: containerID) + try await generator.doesPathExist(containerLib64, inContainer: containerID) { let sdkUsrLib64Path = sdkUsrPath.appending("lib64") // we already checked that the path exists above, so we don't pass `failIfNotExists: false` here. @@ -93,10 +93,10 @@ extension SwiftSDKGenerator { // Copy the ELF interpreter try await generator.copyFromDockerContainer( - id: containerID, - from: FilePath(targetTriple.interpreterPath), - to: sdkDirPath.appending(targetTriple.interpreterPath), - failIfNotExists: true + id: containerID, + from: FilePath(targetTriple.interpreterPath), + to: sdkDirPath.appending(targetTriple.interpreterPath), + failIfNotExists: true ) // Python artifacts are redundant. @@ -119,7 +119,9 @@ extension SwiftSDKGenerator { let fromPath = distributionPath.appending(pathWithinPackage) if isOptional && !doesFileExist(at: fromPath) { - logger.debug("Optional package path ignored since it does not exist", metadata: ["packagePath": .string(fromPath.string)]) + logger.debug( + "Optional package path ignored since it does not exist", + metadata: ["packagePath": .string(fromPath.string)]) continue } @@ -131,10 +133,10 @@ extension SwiftSDKGenerator { extension Triple { var interpreterPath: String { switch self.archName { - case "x86_64": return "/lib64/ld-linux-x86-64.so.2" - case "aarch64": return "/lib/ld-linux-aarch64.so.1" - case "armv7": return "/lib/ld-linux-armhf.so.3" - default: fatalError("unsupported architecture \(self.archName)") + case "x86_64": return "/lib64/ld-linux-x86-64.so.2" + case "aarch64": return "/lib/ld-linux-aarch64.so.1" + case "armv7": return "/lib/ld-linux-armhf.so.3" + default: fatalError("unsupported architecture \(self.archName)") } } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 59db653..ad84680 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -11,14 +11,13 @@ //===----------------------------------------------------------------------===// import AsyncAlgorithms -import Logging import Helpers +import Logging import RegexBuilder import class Foundation.ByteCountFormatter import class Foundation.FileManager import struct Foundation.URL - import struct SystemPackage.FilePath private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" @@ -40,7 +39,9 @@ extension SwiftSDKGenerator { let hostLLVMURL = downloadableArtifacts.hostLLVM.remoteURL // Workaround an issue with github.com returning 400 instead of 404 status to HEAD requests from AHC. - if itemsToDownload(downloadableArtifacts).contains(where: { $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL } ) { + if itemsToDownload(downloadableArtifacts).contains(where: { + $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL + }) { let isLLVMBinaryArtifactAvailable = try await type(of: client).with(http1Only: true) { try await $0.head( url: hostLLVMURL.absoluteString, @@ -56,7 +57,8 @@ extension SwiftSDKGenerator { let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in for item in itemsToDownload(downloadableArtifacts) { group.addTask { - try await engine[DownloadArtifactQuery(artifact: item, httpClient: client, logger: self.logger)] + try await engine[ + DownloadArtifactQuery(artifact: item, httpClient: client, logger: self.logger)] } } @@ -68,9 +70,11 @@ extension SwiftSDKGenerator { } logger.info("Using downloaded artifacts from cache") - logger.debug("Using downloaded artifacts in these locations.", metadata: [ - "paths": .array(results.map(\.path.metadataValue)) - ]) + logger.debug( + "Using downloaded artifacts in these locations.", + metadata: [ + "paths": .array(results.map(\.path.metadataValue)) + ]) } func downloadUbuntuPackages( @@ -85,10 +89,11 @@ extension SwiftSDKGenerator { // Find xz path let xzPath = try await which("xz") if xzPath == nil { - logger.warning(""" - The `xz` utility was not found in `PATH`. \ - Consider installing it for more efficient downloading of package lists. - """) + logger.warning( + """ + The `xz` utility was not found in `PATH`. \ + Consider installing it for more efficient downloading of package lists. + """) } async let mainPackages = try await client.parseUbuntuPackagesList( @@ -117,7 +122,8 @@ extension SwiftSDKGenerator { xzPath: xzPath ) - let allPackages = try await mainPackages + let allPackages = + try await mainPackages .merging(updatesPackages, uniquingKeysWith: { $1 }) .merging(universePackages, uniquingKeysWith: { $1 }) @@ -130,7 +136,8 @@ extension SwiftSDKGenerator { ) } - logger.info("Downloading Ubuntu packages...", metadata: ["packageCount": .stringConvertible(urls.count)]) + logger.info( + "Downloading Ubuntu packages...", metadata: ["packageCount": .stringConvertible(urls.count)]) try await inTemporaryDirectory { fs, tmpDir in let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine) await report(downloadedFiles: downloadedFiles) @@ -151,13 +158,16 @@ extension SwiftSDKGenerator { try await withThrowingTaskGroup(of: (URL, UInt64).self) { for url in urls { $0.addTask { - let downloadedFilePath = try await engine[DownloadFileQuery( - remoteURL: url, localDirectory: directory, httpClient: client - )] + let downloadedFilePath = try await engine[ + DownloadFileQuery( + remoteURL: url, localDirectory: directory, httpClient: client + )] let filePath = downloadedFilePath.path - guard let fileSize = try FileManager.default.attributesOfItem( - atPath: filePath.string - )[.size] as? UInt64 else { + guard + let fileSize = try FileManager.default.attributesOfItem( + atPath: filePath.string + )[.size] as? UInt64 + else { throw GeneratorError.fileDoesNotExist(filePath) } return (url, fileSize) @@ -176,10 +186,12 @@ extension SwiftSDKGenerator { let byteCountFormatter = ByteCountFormatter() for (url, bytes) in downloadedFiles { - logger.debug("Downloaded package", metadata: [ - "url": .string(url.absoluteString), - "size": .string(byteCountFormatter.string(fromByteCount: Int64(bytes))) - ]) + logger.debug( + "Downloaded package", + metadata: [ + "url": .string(url.absoluteString), + "size": .string(byteCountFormatter.string(fromByteCount: Int64(bytes))), + ]) } } } @@ -190,7 +202,8 @@ extension HTTPClientProtocol { unzipWith zipPath: String, isVerbose: Bool ) async throws -> String? { - guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) else { + guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) + else { throw FileOperationError.downloadFailed(url) } @@ -221,16 +234,18 @@ extension HTTPClientProtocol { } let packagesListURL = """ - \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\( - targetTriple.arch!.debianConventionName - )/\(packagesFileName(isXzAvailable: xzPath != nil)) - """ - - guard let packages = try await downloadUbuntuPackagesList( - from: packagesListURL, - unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available - isVerbose: isVerbose - ) else { + \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\( + targetTriple.arch!.debianConventionName + )/\(packagesFileName(isXzAvailable: xzPath != nil)) + """ + + guard + let packages = try await downloadUbuntuPackagesList( + from: packagesListURL, + unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available + isVerbose: isVerbose + ) + else { throw GeneratorError.ubuntuPackagesDecompressionFailure } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index 8bf18f5..fa95b6b 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -11,17 +11,18 @@ //===----------------------------------------------------------------------===// import AsyncAlgorithms -#if canImport(AsyncHTTPClient) -import AsyncHTTPClient -#endif import Foundation import Helpers import RegexBuilder import SystemPackage -public extension Triple.Arch { +#if canImport(AsyncHTTPClient) + import AsyncHTTPClient +#endif + +extension Triple.Arch { /// Returns the value of `cpu` converted to a convention used in Debian package names - var debianConventionName: String { + public var debianConventionName: String { switch self { case .aarch64: return "arm64" case .x86_64: return "amd64" @@ -32,14 +33,15 @@ public extension Triple.Arch { } } -public extension SwiftSDKGenerator { - func run(recipe: SwiftSDKRecipe) async throws { - try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in +extension SwiftSDKGenerator { + public func run(recipe: SwiftSDKRecipe) async throws { + try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { + engine in let httpClientType: HTTPClientProtocol.Type #if canImport(AsyncHTTPClient) - httpClientType = HTTPClient.self + httpClientType = HTTPClient.self #else - httpClientType = OfflineHTTPClient.self + httpClientType = OfflineHTTPClient.self #endif try await httpClientType.with { client in if !self.isIncremental { @@ -48,7 +50,8 @@ public extension SwiftSDKGenerator { try await self.createDirectoryIfNeeded(at: pathsConfiguration.artifactsCachePath) - let swiftSDKProduct = try await recipe.makeSwiftSDK(generator: self, engine: engine, httpClient: client) + let swiftSDKProduct = try await recipe.makeSwiftSDK( + generator: self, engine: engine, httpClient: client) let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift index 94df821..3161b5e 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift @@ -57,7 +57,8 @@ extension SwiftSDKGenerator { } func symlinkClangHeaders() throws { - let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang") + let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending( + "usr/lib/swift_static/clang") if !doesFileExist(at: swiftStaticClangPath) { logger.info("Symlinking clang headers...") try self.createSymlink( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index 8388617..9a27475 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -43,7 +43,9 @@ extension SwiftSDKGenerator { return toolsetJSONPath } - func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath, recipe: SwiftSDKRecipe) throws { + func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath, recipe: SwiftSDKRecipe) + throws + { logger.info("Generating destination JSON file...") let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending("swift-sdk.json") @@ -57,10 +59,11 @@ extension SwiftSDKGenerator { relativeSDKDir.removePrefix(pathsConfiguration.swiftSDKRootPath), relativeToolsetPath.removePrefix(pathsConfiguration.swiftSDKRootPath) else { - fatalError(""" - `toolchainBinDirPath`, `sdkDirPath`, and `toolsetPath` are at unexpected locations that prevent computing \ - relative paths - """) + fatalError( + """ + `toolchainBinDirPath`, `sdkDirPath`, and `toolsetPath` are at unexpected locations that prevent computing \ + relative paths + """) } var metadata = SwiftSDKMetadataV4.TripleProperties( @@ -79,7 +82,7 @@ extension SwiftSDKGenerator { encoder.encode( SwiftSDKMetadataV4( targetTriples: [ - self.targetTriple.triple: metadata, + self.targetTriple.triple: metadata ] ) ) @@ -104,9 +107,9 @@ extension SwiftSDKGenerator { .init( path: FilePath(artifactID).appending(self.targetTriple.triple).string, supportedTriples: hostTriples.map { $0.map(\.triple) } - ), + ) ] - ), + ) ] ) ) @@ -121,7 +124,7 @@ extension SwiftSDKGenerator { } /// Generates an `SDKSettings.json` file that looks like this: - /// + /// /// ```json /// { /// "CanonicalName" : "-swift-linux-[gnu|gnueabihf]", diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index d025768..563936f 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -51,10 +51,10 @@ extension SwiftSDKGenerator { try self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath) let excludes = - unusedTargetPlatforms.map { "--exclude usr/lib/swift/\($0)" } + - unusedTargetPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } + - unusedHostBinaries.map { "--exclude usr/bin/\($0)" } + - unusedHostLibraries.map { "--exclude usr/lib/\($0)" } + unusedTargetPlatforms.map { "--exclude usr/lib/swift/\($0)" } + + unusedTargetPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } + + unusedHostBinaries.map { "--exclude usr/bin/\($0)" } + + unusedHostLibraries.map { "--exclude usr/lib/\($0)" } if hostSwiftPackagePath.string.contains("tar.gz") { try await Shell.run( @@ -108,7 +108,9 @@ extension SwiftSDKGenerator { } } - func prepareLLDLinker(_ engine: QueryEngine, llvmArtifact: DownloadableArtifacts.Item) async throws { + func prepareLLDLinker(_ engine: QueryEngine, llvmArtifact: DownloadableArtifacts.Item) + async throws + { logger.info("Unpacking and copying `lld` linker...") let pathsConfiguration = self.pathsConfiguration let targetOS = self.targetTriple.os @@ -120,23 +122,27 @@ extension SwiftSDKGenerator { let unpackedLLDPath: FilePath if llvmArtifact.isPrebuilt { - unpackedLLDPath = try await engine[TarExtractQuery( - file: llvmArtifact.localPath, - into: untarDestination, - outputBinarySubpath: ["bin", "lld"], - stripComponents: 1 - )].path + unpackedLLDPath = try await engine[ + TarExtractQuery( + file: llvmArtifact.localPath, + into: untarDestination, + outputBinarySubpath: ["bin", "lld"], + stripComponents: 1 + ) + ].path } else { try await self.untar( file: llvmArtifact.localPath, into: untarDestination, stripComponents: 1 ) - unpackedLLDPath = try await engine[CMakeBuildQuery( - sourcesDirectory: untarDestination, - outputBinarySubpath: ["bin", "lld"], - options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=''" - )].path + unpackedLLDPath = try await engine[ + CMakeBuildQuery( + sourcesDirectory: untarDestination, + outputBinarySubpath: ["bin", "lld"], + options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=''" + ) + ].path } let toolchainLLDPath: FilePath diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index 0db1d5e..06a9b27 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -58,19 +58,21 @@ public actor SwiftSDKGenerator { } private let fileManager = FileManager.default - private static let dockerCommand = ProcessInfo.processInfo.environment["SWIFT_SDK_GENERATOR_CONTAINER_RUNTIME"] ?? "docker" + private static let dockerCommand = + ProcessInfo.processInfo.environment["SWIFT_SDK_GENERATOR_CONTAINER_RUNTIME"] ?? "docker" public static func getCurrentTriple(isVerbose: Bool) throws -> Triple { let current = UnixName.current! let cpu = current.machine #if os(macOS) - let darwinVersion = current.release - let darwinTriple = Triple("\(cpu)-apple-darwin\(darwinVersion)") - return Triple("\(cpu)-apple-macos\(darwinTriple._macOSVersion?.description ?? "")") + let darwinVersion = current.release + let darwinTriple = Triple("\(cpu)-apple-darwin\(darwinVersion)") + return Triple("\(cpu)-apple-macos\(darwinTriple._macOSVersion?.description ?? "")") #elseif os(Linux) - return Triple("\(cpu)-unknown-linux-gnu") + return Triple("\(cpu)-unknown-linux-gnu") #else - fatalError("Triple detection not implemented for the platform that this generator was built on.") + fatalError( + "Triple detection not implemented for the platform that this generator was built on.") #endif } @@ -132,7 +134,7 @@ public actor SwiftSDKGenerator { func withDockerContainer( fromImage imageName: String, - _ body: @Sendable (String) async throws -> () + _ body: @Sendable (String) async throws -> Void ) async throws { let containerID = try await launchDockerContainer(imageName: imageName) try await withAsyncThrowing { @@ -176,19 +178,24 @@ public actor SwiftSDKGenerator { } func findSymlinks(at directory: FilePath) throws -> [(FilePath, FilePath)] { - guard let enumerator = fileManager.enumerator( - at: URL(fileURLWithPath: directory.string), - includingPropertiesForKeys: [.isSymbolicLinkKey] - ) else { return [] } + guard + let enumerator = fileManager.enumerator( + at: URL(fileURLWithPath: directory.string), + includingPropertiesForKeys: [.isSymbolicLinkKey] + ) + else { return [] } var result = [(FilePath, FilePath)]() for case let url as URL in enumerator { - guard let isSymlink = try url.resourceValues(forKeys: [.isSymbolicLinkKey]) - .isSymbolicLink else { continue } + guard + let isSymlink = try url.resourceValues(forKeys: [.isSymbolicLinkKey]) + .isSymbolicLink + else { continue } if isSymlink { let path = url.path - try result.append((FilePath(path), FilePath(self.fileManager.destinationOfSymbolicLink(atPath: url.path)))) + try result.append( + (FilePath(path), FilePath(self.fileManager.destinationOfSymbolicLink(atPath: url.path)))) } } @@ -227,7 +234,8 @@ public actor SwiftSDKGenerator { } func gunzip(file: FilePath, into directoryPath: FilePath) async throws { - try await Shell.run(#"cd "\#(directoryPath)" && gzip -d "\#(file)""#, shouldLogCommands: self.isVerbose) + try await Shell.run( + #"cd "\#(directoryPath)" && gzip -d "\#(file)""#, shouldLogCommands: self.isVerbose) } func untar( @@ -254,7 +262,9 @@ public actor SwiftSDKGenerator { if isVerbose { let cmd = "ls \(tmp)" let lsOutput = try await Shell.readStdout(cmd) - logger.debug("Files unpacked from deb file", metadata: ["cmd": .string(cmd), "output": .string(lsOutput)]) + logger.debug( + "Files unpacked from deb file", + metadata: ["cmd": .string(cmd), "output": .string(lsOutput)]) } try await Shell.run( diff --git a/Sources/SwiftSDKGenerator/PathsConfiguration.swift b/Sources/SwiftSDKGenerator/PathsConfiguration.swift index cb898d0..bd0d12b 100644 --- a/Sources/SwiftSDKGenerator/PathsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PathsConfiguration.swift @@ -15,7 +15,8 @@ import struct SystemPackage.FilePath public struct PathsConfiguration: Sendable { init(sourceRoot: FilePath, artifactID: String, targetTriple: Triple) { self.sourceRoot = sourceRoot - self.artifactBundlePath = sourceRoot + self.artifactBundlePath = + sourceRoot .appending("Bundles") .appending("\(artifactID).artifactbundle") self.artifactsCachePath = sourceRoot.appending("Artifacts") diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index f10b818..0152478 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -34,7 +34,8 @@ public enum LinuxDistribution: Hashable, Sendable { case "24.04": self = .noble default: - throw GeneratorError.unknownLinuxDistribution(name: LinuxDistribution.Name.ubuntu.rawValue, version: version) + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.ubuntu.rawValue, version: version) } } @@ -48,7 +49,8 @@ public enum LinuxDistribution: Hashable, Sendable { public var requiredPackages: [String] { switch self { - case .focal: return [ + case .focal: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -61,7 +63,8 @@ public enum LinuxDistribution: Hashable, Sendable { "zlib1g", "zlib1g-dev", ] - case .jammy: return [ + case .jammy: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -74,7 +77,8 @@ public enum LinuxDistribution: Hashable, Sendable { "zlib1g", "zlib1g-dev", ] - case .noble: return [ + case .noble: + return [ "libc6", "libc6-dev", "libgcc-s1", @@ -129,8 +133,8 @@ public enum LinuxDistribution: Hashable, Sendable { } } -public extension LinuxDistribution.Name { - init(nameString: String) throws { +extension LinuxDistribution.Name { + public init(nameString: String) throws { guard let name = LinuxDistribution.Name(rawValue: nameString) else { throw GeneratorError.unknownLinuxDistribution(name: nameString, version: nil) } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift b/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift index fd7e1c3..55dec55 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/Triple.swift @@ -16,12 +16,12 @@ public typealias Triple = Helpers.Triple extension Triple: @unchecked Sendable {} -public extension Triple { - init(arch: Arch, vendor: Vendor?, os: OS, environment: Environment) { +extension Triple { + public init(arch: Arch, vendor: Vendor?, os: OS, environment: Environment) { self.init("\(arch)-\(vendor?.rawValue ?? "unknown")-\(os)-\(environment)", normalizing: true) } - init(arch: Arch, vendor: Vendor?, os: OS) { + public init(arch: Arch, vendor: Vendor?, os: OS) { self.init("\(arch)-\(vendor?.rawValue ?? "unknown")-\(os)", normalizing: true) } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 85ca59a..88467aa 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -24,7 +24,8 @@ public struct VersionsConfiguration: Sendable { self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" self.lldVersion = lldVersion self.linuxDistribution = linuxDistribution - self.linuxArchSuffix = targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + self.linuxArchSuffix = + targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" } let swiftVersion: String @@ -54,19 +55,20 @@ public struct VersionsConfiguration: Sendable { let computedSubdirectory: String switch self.linuxDistribution { case let .ubuntu(ubuntu): - computedSubdirectory = "ubuntu\(ubuntu.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" + computedSubdirectory = + "ubuntu\(ubuntu.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" case let .rhel(rhel): computedSubdirectory = rhel.rawValue } return URL( string: """ - https://download.swift.org/\( - self.swiftBranch - )/\( - subdirectory ?? computedSubdirectory - )/swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: platform)).\(fileExtension) - """ + https://download.swift.org/\( + self.swiftBranch + )/\( + subdirectory ?? computedSubdirectory + )/swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: platform)).\(fileExtension) + """ )! } diff --git a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift index a5d5e20..d0d8652 100644 --- a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import Helpers + import struct SystemPackage.FilePath struct CMakeBuildQuery: CachingQuery { diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift index 041737c..e81d2c3 100644 --- a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift @@ -10,9 +10,10 @@ // //===----------------------------------------------------------------------===// -import class Foundation.ByteCountFormatter -import Logging import Helpers +import Logging + +import class Foundation.ByteCountFormatter import struct SystemPackage.FilePath struct DownloadArtifactQuery: Query { @@ -22,7 +23,9 @@ struct DownloadArtifactQuery: Query { let logger: Logger func run(engine: QueryEngine) async throws -> FilePath { - logger.info("Downloading remote artifact not available in local cache", metadata: ["remoteUrl": .string(self.artifact.remoteURL.absoluteString)]) + logger.info( + "Downloading remote artifact not available in local cache", + metadata: ["remoteUrl": .string(self.artifact.remoteURL.absoluteString)]) let stream = self.httpClient.streamDownloadProgress( from: self.artifact.remoteURL, to: self.artifact.localPath ) @@ -39,15 +42,16 @@ struct DownloadArtifactQuery: Query { let byteCountFormatter = ByteCountFormatter() if let total = progress.totalBytes { - logger.debug(""" - \(artifact.remoteURL.lastPathComponent) \( - byteCountFormatter - .string(fromByteCount: Int64(progress.receivedBytes)) - )/\( - byteCountFormatter - .string(fromByteCount: Int64(total)) - ) - """) + logger.debug( + """ + \(artifact.remoteURL.lastPathComponent) \( + byteCountFormatter + .string(fromByteCount: Int64(progress.receivedBytes)) + )/\( + byteCountFormatter + .string(fromByteCount: Int64(total)) + ) + """) } else { logger.debug( "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))" diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift index 41d9a1a..43e3d4e 100644 --- a/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift @@ -10,8 +10,9 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.URL import Helpers + +import struct Foundation.URL import struct SystemPackage.FilePath struct DownloadFileQuery: Query { diff --git a/Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift b/Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift index cf5828d..4eeabf5 100644 --- a/Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2022-202Apple Inc. and the Swift project authors +// Copyright (c) 2022-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 66a4dfe..7d58357 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -11,8 +11,9 @@ //===----------------------------------------------------------------------===// import Foundation -import Logging import Helpers +import Logging + import struct SystemPackage.FilePath public struct LinuxRecipe: SwiftSDKRecipe { @@ -177,7 +178,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { var items: [DownloadableArtifacts.Item] = [] if self.hostSwiftSource != .preinstalled && self.mainHostTriple.os != .linux - && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") { + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") + { items.append(artifacts.hostLLVM) } @@ -226,7 +228,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { self.linuxDistribution, targetArchName: self.mainTargetTriple.archName ) } - + let sdkDirPath = self.sdkDirPath(paths: generator.pathsConfiguration) if !generator.isIncremental { try await generator.removeRecursively(at: sdkDirPath) @@ -249,7 +251,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { if !self.shouldUseDocker { guard case let .ubuntu(version) = linuxDistribution else { - throw GeneratorError + throw + GeneratorError .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) } @@ -289,7 +292,9 @@ public struct LinuxRecipe: SwiftSDKRecipe { case .remoteTarball: try await generator.unpackTargetSwiftPackage( targetSwiftPackagePath: downloadableArtifacts.targetSwift.localPath, - relativePathToRoot: [FilePath.Component(self.versionsConfiguration.swiftDistributionName())!], + relativePathToRoot: [ + FilePath.Component(self.versionsConfiguration.swiftDistributionName())! + ], sdkDirPath: sdkDirPath ) } @@ -308,11 +313,14 @@ public struct LinuxRecipe: SwiftSDKRecipe { // Swift 6.1 and later do not throw warnings about the SDKSettings.json file missing, // so they don't need this file. if self.versionsConfiguration.swiftVersion.hasAnyPrefix(from: ["5.9", "5.10", "6.0"]) { - try await generator.generateSDKSettingsFile(sdkDirPath: sdkDirPath, distribution: linuxDistribution) + try await generator.generateSDKSettingsFile( + sdkDirPath: sdkDirPath, distribution: linuxDistribution) } if self.hostSwiftSource != .preinstalled { - if self.mainHostTriple.os != .linux && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") { + if self.mainHostTriple.os != .linux + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") + { try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM) } @@ -320,7 +328,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { try await generator.symlinkClangHeaders() } - let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending("swift-autolink-extract") + let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending( + "swift-autolink-extract") if await !generator.doesFileExist(at: autolinkExtractPath) { logger.info("Fixing `swift-autolink-extract` symlink...") diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 17908a4..c10f008 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -10,8 +10,9 @@ // //===----------------------------------------------------------------------===// -import Logging import Helpers +import Logging + import struct SystemPackage.FilePath public struct SwiftSDKProduct { @@ -39,13 +40,15 @@ public protocol SwiftSDKRecipe: Sendable { var logger: Logger { get } /// The main entrypoint of the recipe to make a Swift SDK - func makeSwiftSDK(generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol) async throws + func makeSwiftSDK( + generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol + ) async throws -> SwiftSDKProduct } -public extension SwiftSDKRecipe { - func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) {} - func applyPlatformOptions( +extension SwiftSDKRecipe { + public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) {} + public func applyPlatformOptions( metadata: inout SwiftSDKMetadataV4.TripleProperties, paths: PathsConfiguration, targetTriple: Triple diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index 21b2485..dc3cb58 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -10,8 +10,9 @@ // //===----------------------------------------------------------------------===// -import Logging import Helpers +import Logging + import struct SystemPackage.FilePath public struct WebAssemblyRecipe: SwiftSDKRecipe { @@ -59,9 +60,10 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { "-pthread", "-ftls-model=local-exec", ] // Tell LLVM codegen in swiftc to enable those features via clang options - toolset.swiftCompiler?.extraCLIOptions?.append(contentsOf: ccOptions.flatMap { - ["-Xcc", $0] - }) + toolset.swiftCompiler?.extraCLIOptions?.append( + contentsOf: ccOptions.flatMap { + ["-Xcc", $0] + }) // Tell the C and C++ compilers to enable those features toolset.cCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) toolset.cxxCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) @@ -86,9 +88,11 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { ) { var relativeToolchainDir = paths.toolchainDirPath guard relativeToolchainDir.removePrefix(paths.swiftSDKRootPath) else { - fatalError("The toolchain bin directory path must be a subdirectory of the Swift SDK root path.") + fatalError( + "The toolchain bin directory path must be a subdirectory of the Swift SDK root path.") } - metadata.swiftStaticResourcesPath = relativeToolchainDir.appending("usr/lib/swift_static").string + metadata.swiftStaticResourcesPath = + relativeToolchainDir.appending("usr/lib/swift_static").string metadata.swiftResourcesPath = metadata.swiftStaticResourcesPath } @@ -104,7 +108,8 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { var hostTriples: [Triple]? = nil if let hostSwiftPackage { hostTriples = [hostSwiftPackage.triple] - try await generator.rsync(from: hostSwiftPackage.path.appending("usr"), to: pathsConfiguration.toolchainDirPath) + try await generator.rsync( + from: hostSwiftPackage.path.appending("usr"), to: pathsConfiguration.toolchainDirPath) logger.info("Removing unused toolchain components...") let liblldbNames: [String] = try await { @@ -127,15 +132,19 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { try await self.mergeTargetSwift(from: targetSwiftLibPath, generator: generator) } else { // Simply copy the target Swift package into the SDK bundle when building host-agnostic SDK. - try await generator.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath.appending("usr")) - try await generator.copy(from: targetSwiftLibPath, to: pathsConfiguration.toolchainDirPath.appending("usr/lib")) + try await generator.createDirectoryIfNeeded( + at: pathsConfiguration.toolchainDirPath.appending("usr")) + try await generator.copy( + from: targetSwiftLibPath, to: pathsConfiguration.toolchainDirPath.appending("usr/lib")) } - let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending("swift-autolink-extract") + let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending( + "swift-autolink-extract") // WebAssembly object file requires `swift-autolink-extract` if await !generator.doesFileExist(at: autolinkExtractPath), - await generator.doesFileExist(at: generator.pathsConfiguration.toolchainBinDirPath.appending("swift")) + await generator.doesFileExist( + at: generator.pathsConfiguration.toolchainBinDirPath.appending("swift")) { logger.info("Fixing `swift-autolink-extract` symlink...") try await generator.createSymlink(at: autolinkExtractPath, pointingTo: "swift") @@ -149,24 +158,40 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } /// Merge the target Swift package into the Swift SDK bundle derived from the host Swift package. - func mergeTargetSwift(from distributionPath: FilePath, generator: SwiftSDKGenerator) async throws { + func mergeTargetSwift(from distributionPath: FilePath, generator: SwiftSDKGenerator) async throws + { let pathsConfiguration = generator.pathsConfiguration logger.info("Copying Swift core libraries for the target triple into Swift SDK bundle...") for (pathWithinPackage, pathWithinSwiftSDK, isOptional) in [ ("clang", pathsConfiguration.toolchainDirPath.appending("usr/lib"), false), ("swift/clang", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift"), false), ("swift/wasi", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift"), false), - ("swift_static/clang", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), false), - ("swift_static/wasi", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), false), - ("swift_static/shims", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), false), + ( + "swift_static/clang", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), + false + ), + ( + "swift_static/wasi", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), + false + ), + ( + "swift_static/shims", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), + false + ), // Mark CoreFoundation as optional until we set up build system to build it for WebAssembly - ("swift_static/CoreFoundation", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), true), + ( + "swift_static/CoreFoundation", + pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static"), true + ), ] { - if isOptional, await !(generator.doesFileExist(at: distributionPath.appending(pathWithinPackage))) { + if isOptional, + await !(generator.doesFileExist(at: distributionPath.appending(pathWithinPackage))) + { logger.debug("Skipping optional path \(pathWithinPackage)") continue } - try await generator.rsync(from: distributionPath.appending(pathWithinPackage), to: pathWithinSwiftSDK) + try await generator.rsync( + from: distributionPath.appending(pathWithinPackage), to: pathWithinSwiftSDK) } } } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index f16b82a..7dca815 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -14,8 +14,8 @@ import AsyncProcess import Foundation import NIOCore -public extension ByteBuffer { - func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { +extension ByteBuffer { + public func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { let result = try await ProcessExecutor.runCollectingOutput( executable: zipPath, ["-cd"], standardInput: [self].async, diff --git a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift index 2ecae65..e7706de 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift @@ -39,7 +39,8 @@ extension GeneratorError: CustomStringConvertible { case let .nonZeroExitCode(exitCode, commandInfo): return "Process launched with \(commandInfo) failed with exit code \(exitCode)" case let .unknownLinuxDistribution(name, version): - return "Linux distribution `\(name)`\(version.map { " with version \($0)" } ?? "")` is not supported by this generator." + return + "Linux distribution `\(name)`\(version.map { " with version \($0)" } ?? "")` is not supported by this generator." case let .unknownMacOSVersion(version): return "macOS version `\(version)` is not supported by this generator." case let .unknownCPUArchitecture(cpu): @@ -48,21 +49,23 @@ extension GeneratorError: CustomStringConvertible { return "LLD version `\(version)` is not supported by this generator." case let .distributionSupportsOnlyDockerGenerator(linuxDistribution): return """ - Target Linux distribution \(linuxDistribution) supports Swift SDK generation only when `--with-docker` flag is \ - passed. - """ + Target Linux distribution \(linuxDistribution) supports Swift SDK generation only when `--with-docker` flag is \ + passed. + """ case let .distributionDoesNotSupportArchitecture(linuxDistribution, targetArchName): return """ - Target Linux distribution \(linuxDistribution) does not support the target architecture: \(targetArchName) - """ + Target Linux distribution \(linuxDistribution) does not support the target architecture: \(targetArchName) + """ case let .fileDoesNotExist(filePath): return "Expected to find a file at path `\(filePath)`." case let .fileDownloadFailed(url, status): - return "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." + return + "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." case .ubuntuPackagesDecompressionFailure: return "Failed to decompress the list of Ubuntu packages" case let .ubuntuPackagesParsingFailure(expected, actual): - return "Failed to parse Ubuntu packages manifest, expected \(expected), found \(actual) packages." + return + "Failed to parse Ubuntu packages manifest, expected \(expected), found \(actual) packages." } } } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift b/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift index a5abfeb..b11d02e 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift @@ -53,7 +53,9 @@ public protocol HTTPClientProtocol: Sendable { } extension HTTPClientProtocol { - static func with(_ body: @Sendable (any HTTPClientProtocol) async throws -> Result) async throws + static func with( + _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + ) async throws -> Result { try await self.with(http1Only: false, body) @@ -63,107 +65,112 @@ extension HTTPClientProtocol { extension FilePath: @unchecked Sendable {} #if canImport(AsyncHTTPClient) -import AsyncHTTPClient - -extension FileDownloadDelegate.Progress: @unchecked Sendable {} - -extension HTTPClient: HTTPClientProtocol { - public static func with( - http1Only: Bool, _ body: @Sendable (any HTTPClientProtocol) async throws -> Result - ) async throws -> Result { - var configuration = HTTPClient.Configuration(redirectConfiguration: .follow(max: 5, allowCycles: false)) - if http1Only { - configuration.httpVersion = .http1Only - } - let client = HTTPClient(eventLoopGroupProvider: .singleton, configuration: configuration) - return try await withAsyncThrowing { - try await body(client) - } defer: { - try await client.shutdown() + import AsyncHTTPClient + + extension FileDownloadDelegate.Progress: @unchecked Sendable {} + + extension HTTPClient: HTTPClientProtocol { + public static func with( + http1Only: Bool, _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + ) async throws -> Result { + var configuration = HTTPClient.Configuration( + redirectConfiguration: .follow(max: 5, allowCycles: false)) + if http1Only { + configuration.httpVersion = .http1Only + } + let client = HTTPClient(eventLoopGroupProvider: .singleton, configuration: configuration) + return try await withAsyncThrowing { + try await body(client) + } defer: { + try await client.shutdown() + } } - } - public func get(url: String) async throws -> (status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer?) { - let response = try await self.get(url: url).get() - return (status: response.status, body: response.body) - } + public func get(url: String) async throws -> ( + status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer? + ) { + let response = try await self.get(url: url).get() + return (status: response.status, body: response.body) + } - public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { - var headRequest = HTTPClientRequest(url: url) - headRequest.method = .HEAD - headRequest.headers = ["Accept": "*/*", "User-Agent": "Swift SDK Generator"] - return try await self.execute(headRequest, deadline: .distantFuture).status == .ok - } + public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { + var headRequest = HTTPClientRequest(url: url) + headRequest.method = .HEAD + headRequest.headers = ["Accept": "*/*", "User-Agent": "Swift SDK Generator"] + return try await self.execute(headRequest, deadline: .distantFuture).status == .ok + } - public func downloadFile( - from url: URL, - to path: FilePath - ) async throws { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<(), Error>) in - do { - let delegate = try FileDownloadDelegate( - path: path.string, - reportHead: { task, responseHead in - if responseHead.status != .ok { - task.fail(reason: GeneratorError.fileDownloadFailed(url, responseHead.status.description)) + public func downloadFile( + from url: URL, + to path: FilePath + ) async throws { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<(), Error>) in + do { + let delegate = try FileDownloadDelegate( + path: path.string, + reportHead: { task, responseHead in + if responseHead.status != .ok { + task.fail( + reason: GeneratorError.fileDownloadFailed(url, responseHead.status.description)) + } + } + ) + let request = try HTTPClient.Request(url: url) + + execute(request: request, delegate: delegate).futureResult.whenComplete { + switch $0 { + case let .failure(error): + continuation.resume(throwing: error) + case .success: + continuation.resume(returning: ()) } } - ) - let request = try HTTPClient.Request(url: url) - - execute(request: request, delegate: delegate).futureResult.whenComplete { - switch $0 { - case let .failure(error): - continuation.resume(throwing: error) - case .success: - continuation.resume(returning: ()) - } + } catch { + continuation.resume(throwing: error) } - } catch { - continuation.resume(throwing: error) } } - } - public func streamDownloadProgress( - from url: URL, - to path: FilePath - ) -> AsyncThrowingStream { - .init { continuation in - do { - let delegate = try FileDownloadDelegate( - path: path.string, - reportHead: { - if $0.status != .ok { - continuation - .finish(throwing: FileOperationError.downloadFailed(url, $0.status.description)) + public func streamDownloadProgress( + from url: URL, + to path: FilePath + ) -> AsyncThrowingStream { + .init { continuation in + do { + let delegate = try FileDownloadDelegate( + path: path.string, + reportHead: { + if $0.status != .ok { + continuation + .finish(throwing: FileOperationError.downloadFailed(url, $0.status.description)) + } + }, + reportProgress: { + continuation.yield( + DownloadProgress(totalBytes: $0.totalBytes, receivedBytes: $0.receivedBytes) + ) + } + ) + let request = try HTTPClient.Request(url: url) + + execute(request: request, delegate: delegate).futureResult.whenComplete { + switch $0 { + case let .failure(error): + continuation.finish(throwing: error) + case let .success(finalProgress): + continuation.yield( + DownloadProgress( + totalBytes: finalProgress.totalBytes, receivedBytes: finalProgress.receivedBytes) + ) + continuation.finish() } - }, - reportProgress: { - continuation.yield( - DownloadProgress(totalBytes: $0.totalBytes, receivedBytes: $0.receivedBytes) - ) - } - ) - let request = try HTTPClient.Request(url: url) - - execute(request: request, delegate: delegate).futureResult.whenComplete { - switch $0 { - case let .failure(error): - continuation.finish(throwing: error) - case let .success(finalProgress): - continuation.yield( - DownloadProgress(totalBytes: finalProgress.totalBytes, receivedBytes: finalProgress.receivedBytes) - ) - continuation.finish() } + } catch { + continuation.finish(throwing: error) } - } catch { - continuation.finish(throwing: error) } } } -} #endif struct OfflineHTTPClient: HTTPClientProtocol { @@ -189,11 +196,15 @@ struct OfflineHTTPClient: HTTPClientProtocol { } } - public func get(url: String) async throws -> (status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer?) { - throw FileOperationError.downloadFailed(URL(string: url)!, "Cannot fetch file with offline client") + public func get(url: String) async throws -> ( + status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer? + ) { + throw FileOperationError.downloadFailed( + URL(string: url)!, "Cannot fetch file with offline client") } public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { - throw FileOperationError.downloadFailed(URL(string: url)!, "Cannot fetch file with offline client") + throw FileOperationError.downloadFailed( + URL(string: url)!, "Cannot fetch file with offline client") } } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/Shell.swift b/Sources/SwiftSDKGenerator/SystemUtils/Shell.swift index de62dfb..62bc1ca 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/Shell.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/Shell.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import AsyncProcess + import class Foundation.ProcessInfo import struct SystemPackage.FilePath @@ -125,7 +126,9 @@ struct Shell { try result.exitReason.throwIfNonZero() - guard let stdOutBuffer = result.standardOutput else { throw GeneratorError.noProcessOutput(command) } + guard let stdOutBuffer = result.standardOutput else { + throw GeneratorError.noProcessOutput(command) + } return String(buffer: stdOutBuffer) } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift b/Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift index 3d9135c..9389e6f 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift @@ -1,7 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024-2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + #if canImport(Darwin) -import Darwin + import Darwin #elseif canImport(Glibc) -import Glibc + import Glibc #endif /// libc's `uname` wrapper diff --git a/Sources/SwiftSDKGenerator/SystemUtils/which.swift b/Sources/SwiftSDKGenerator/SystemUtils/which.swift index 472c32b..040a746 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/which.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/which.swift @@ -14,13 +14,14 @@ import AsyncProcess import Foundation /// Look for an executable using the `which` utility. -/// +/// /// - Parameter executableName: The name of the executable to search for. /// - Throws: Any errors thrown by the ProcessExecutor. /// - Returns: The path to the executable if found, otherwise nil. func which(_ executableName: String) async throws -> String? { let result = try await ProcessExecutor.runCollectingOutput( - executable: "/usr/bin/which", [executableName], collectStandardOutput: true, collectStandardError: false, + executable: "/usr/bin/which", [executableName], collectStandardOutput: true, + collectStandardError: false, environment: ProcessInfo.processInfo.environment ) diff --git a/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift b/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift index 1ce58a5..4f56cf0 100644 --- a/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift +++ b/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift @@ -43,7 +43,8 @@ final class AsyncByteBufferLineSequenceTests: XCTestCase { func testManyChunksNoNewlineNotDeliveringLastChunk() async throws { for n in 0..<100 { let inputs: [ByteBuffer] = [ByteBuffer(repeating: 0, count: n)] - let lines = try await Array(inputs.async.splitIntoLines(dropLastChunkIfNoNewline: true).strings) + let lines = try await Array( + inputs.async.splitIntoLines(dropLastChunkIfNoNewline: true).strings) XCTAssertEqual([], lines) } } @@ -61,11 +62,11 @@ final class AsyncByteBufferLineSequenceTests: XCTestCase { } func testOverlyLongLineIsSplitByDefault() async throws { - var inputs = [ByteBuffer(repeating: UInt8(0), count: 1024 * 1024 - 2)] // almost at the limit + var inputs = [ByteBuffer(repeating: UInt8(0), count: 1024 * 1024 - 2)] // almost at the limit inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) - inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // hitting the limit - inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // over the limit - inputs.append(ByteBuffer(integer: UInt8(ascii: "\n"))) // too late + inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // hitting the limit + inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // over the limit + inputs.append(ByteBuffer(integer: UInt8(ascii: "\n"))) // too late let lines = try await Array( inputs.async.splitIntoLines( dropTerminator: false, diff --git a/Tests/AsyncProcessTests/IntegrationTests.swift b/Tests/AsyncProcessTests/IntegrationTests.swift index 7b5a4ee..50d3c8a 100644 --- a/Tests/AsyncProcessTests/IntegrationTests.swift +++ b/Tests/AsyncProcessTests/IntegrationTests.swift @@ -19,9 +19,9 @@ import NIOConcurrencyHelpers import XCTest #if canImport(Darwin) -import Darwin + import Darwin #else -import Glibc + import Glibc #endif final class IntegrationTests: XCTestCase { @@ -69,10 +69,10 @@ final class IntegrationTests: XCTestCase { func testSignalsWork() async throws { #if os(Linux) - // workaround for https://github.com/apple/swift-corelibs-foundation/issues/4772 - let signalsToTest: [CInt] = [SIGKILL] + // workaround for https://github.com/apple/swift-corelibs-foundation/issues/4772 + let signalsToTest: [CInt] = [SIGKILL] #else - let signalsToTest: [CInt] = [SIGKILL, SIGTERM, SIGINT] + let signalsToTest: [CInt] = [SIGKILL, SIGTERM, SIGINT] #endif for signal in signalsToTest { let exe = ProcessExecutor( @@ -99,7 +99,7 @@ final class IntegrationTests: XCTestCase { let input = AsyncStream.justMakeIt(elementType: ByteBuffer.self) let exe = ProcessExecutor( group: self.group, - executable: "/bin/cat", ["-nu"], // sh", ["-c", "while read -r line; do echo $line; done"], + executable: "/bin/cat", ["-nu"], // sh", ["-c", "while read -r line; do echo $line; done"], standardInput: input.consumer, logger: self.logger ) @@ -521,7 +521,7 @@ final class IntegrationTests: XCTestCase { func testLogOutputToMetadata() async throws { let sharedRecorder = LogRecorderHandler() var recordedLogger = Logger(label: "recorder", factory: { _ in sharedRecorder }) - recordedLogger.logLevel = .info // don't give us the normal messages + recordedLogger.logLevel = .info // don't give us the normal messages recordedLogger[metadataKey: "yo"] = "hey" try await ProcessExecutor.runLogOutput( @@ -530,20 +530,22 @@ final class IntegrationTests: XCTestCase { ["-c", "echo 1; echo >&2 2; echo 3; echo >&2 4; echo 5; echo >&2 6; echo 7; echo >&2 8;"], standardInput: EOFSequence(), logger: recordedLogger, - logConfiguration: OutputLoggingSettings(logLevel: .critical, to: .metadata(logMessage: "msg", key: "key")) + logConfiguration: OutputLoggingSettings( + logLevel: .critical, to: .metadata(logMessage: "msg", key: "key")) ).throwIfNonZero() XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.level == .critical }) XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.message == "msg" }) XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.metadata["key"] != nil }) XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.metadata["yo"] == "hey" }) - let loggedLines = sharedRecorder.recordedMessages.compactMap { $0.metadata["key"]?.description }.sorted() + let loggedLines = sharedRecorder.recordedMessages.compactMap { $0.metadata["key"]?.description } + .sorted() XCTAssertEqual(["1", "2", "3", "4", "5", "6", "7", "8"], loggedLines) } func testLogOutputToMessage() async throws { let sharedRecorder = LogRecorderHandler() var recordedLogger = Logger(label: "recorder", factory: { _ in sharedRecorder }) - recordedLogger.logLevel = .info // don't give us the normal messages + recordedLogger.logLevel = .info // don't give us the normal messages recordedLogger[metadataKey: "yo"] = "hey" try await ProcessExecutor.runLogOutput( @@ -633,7 +635,8 @@ final class IntegrationTests: XCTestCase { func testCollectJustStandardError() async throws { let allInfo = try await ProcessExecutor.runCollectingOutput( group: self.group, - executable: "/bin/sh", ["-c", "/bin/dd >&2 if=/dev/zero bs=\(1024 * 1024) count=1 status=none"], + executable: "/bin/sh", + ["-c", "/bin/dd >&2 if=/dev/zero bs=\(1024 * 1024) count=1 status=none"], standardInput: EOFSequence(), collectStandardOutput: false, collectStandardError: true, @@ -648,7 +651,8 @@ final class IntegrationTests: XCTestCase { func testCollectNothing() async throws { let allInfo = try await ProcessExecutor.runCollectingOutput( group: self.group, - executable: "/bin/sh", ["-c", "/bin/dd >&2 if=/dev/zero bs=\(1024 * 1024) count=100 status=none"], + executable: "/bin/sh", + ["-c", "/bin/dd >&2 if=/dev/zero bs=\(1024 * 1024) count=100 status=none"], standardInput: EOFSequence(), collectStandardOutput: false, collectStandardError: false, @@ -757,8 +761,8 @@ final class IntegrationTests: XCTestCase { } catch { XCTAssertEqual(NSCocoaErrorDomain, (error as NSError).domain) #if canImport(Darwin) - // https://github.com/apple/swift-corelibs-foundation/issues/4810 - XCTAssertEqual(NSFileNoSuchFileError, (error as NSError).code) + // https://github.com/apple/swift-corelibs-foundation/issues/4810 + XCTAssertEqual(NSFileNoSuchFileError, (error as NSError).code) #endif } } @@ -811,23 +815,23 @@ final class IntegrationTests: XCTestCase { let exeStream = ProcessExecutor(executable: "/bin/sh", ["-c", "true"]) #if compiler(>=5.8) - async let stdout = Array(exeStream.standardOutput) - async let stderr = Array(exeStream.standardError) + async let stdout = Array(exeStream.standardOutput) + async let stderr = Array(exeStream.standardError) #else - async let stdout = { - var chunks: [ByteBuffer] = [] - for try await chunk in await exeStream.standardOutput { - chunks.append(chunk) - } - return chunks - }() - async let stderr = { - var chunks: [ByteBuffer] = [] - for try await chunk in await exeStream.standardError { - chunks.append(chunk) - } - return chunks - }() + async let stdout = { + var chunks: [ByteBuffer] = [] + for try await chunk in await exeStream.standardOutput { + chunks.append(chunk) + } + return chunks + }() + async let stderr = { + var chunks: [ByteBuffer] = [] + for try await chunk in await exeStream.standardError { + chunks.append(chunk) + } + return chunks + }() #endif try await exeStream.run().throwIfNonZero() let out = try await stdout @@ -863,7 +867,9 @@ final class IntegrationTests: XCTestCase { XCTAssertNoThrow(try FileManager.default.removeItem(at: tempDir)) } - for (stdoutMode, stderrMode) in [("shared", "shared"), ("shared", "owned"), ("owned", "shared")] { + for (stdoutMode, stderrMode) in [ + ("shared", "shared"), ("shared", "owned"), ("owned", "shared"), + ] { let filePath = tempDir.appendingPathComponent("file-\(stdoutMode)-\(stderrMode)") let fd = try FileDescriptor.open( .init(filePath.path.removingPercentEncoding!), @@ -892,14 +898,14 @@ final class IntegrationTests: XCTestCase { } #if canImport(Darwin) - let command = - "for o in 1 2; do i=1000; while [ $i -gt 0 ]; do echo $o >&$o; i=$(( $i - 1 )); done & done; wait" + let command = + "for o in 1 2; do i=1000; while [ $i -gt 0 ]; do echo $o >&$o; i=$(( $i - 1 )); done & done; wait" #else - // workaround for - // https://github.com/apple/swift-corelibs-foundation/issues/4772 - // which causes `SIGCHLD` being blocked in the shell so it can't wait for its children :| - let command = - "for o in 1 2; do i=1000; while [ $i -gt 0 ]; do echo $o >&$o; i=$(( $i - 1 )); done & done; sleep 10" + // workaround for + // https://github.com/apple/swift-corelibs-foundation/issues/4772 + // which causes `SIGCHLD` being blocked in the shell so it can't wait for its children :| + let command = + "for o in 1 2; do i=1000; while [ $i -gt 0 ]; do echo $o >&$o; i=$(( $i - 1 )); done & done; sleep 10" #endif let exe = ProcessExecutor( @@ -1020,71 +1026,71 @@ final class IntegrationTests: XCTestCase { } #if os(macOS) - // This test will hang on anything that uses swift-corelibs-foundation because of - // https://github.com/apple/swift-corelibs-foundation/issues/4795 - // Foundation.Process on Linux doesn't correctly detect when child process dies (creating zombie processes) - func testCanDealWithRunawayChildProcesses() async throws { - self.logger = Logger(label: "x") - self.logger.logLevel = .info - let p = ProcessExecutor( - executable: "/bin/bash", - [ - "-c", - """ - set -e - /usr/bin/yes "Runaway process from \(#function), please file a swift-sdk-generator bug." > /dev/null & - child_pid=$! - trap "echo >&2 killing $child_pid; kill -KILL $child_pid" INT - echo "$child_pid" # communicate the child pid to our parent - exec >&- # close stdout - echo "waiting for $child_pid" >&2 - wait - """, - ], - standardError: .discard, - teardownSequence: [ - .sendSignal(SIGINT, allowedTimeToExitNS: 10_000_000_000), - ], - logger: self.logger - ) - - try await withThrowingTaskGroup(of: pid_t?.self) { group in - group.addTask { - let result = try await p.run() - XCTAssertEqual(.exit(128 + SIGINT), result) - return nil - } + // This test will deadlock on anything that uses swift-corelibs-foundation because of + // https://github.com/apple/swift-corelibs-foundation/issues/4795 + // Foundation.Process on Linux doesn't correctly detect when child process dies (creating zombie processes) + func testCanDealWithRunawayChildProcesses() async throws { + self.logger = Logger(label: "x") + self.logger.logLevel = .info + let p = ProcessExecutor( + executable: "/bin/bash", + [ + "-c", + """ + set -e + /usr/bin/yes "Runaway process from \(#function), please file a swift-sdk-generator bug." > /dev/null & + child_pid=$! + trap "echo >&2 killing $child_pid; kill -KILL $child_pid" INT + echo "$child_pid" # communicate the child pid to our parent + exec >&- # close stdout + echo "waiting for $child_pid" >&2 + wait + """, + ], + standardError: .discard, + teardownSequence: [ + .sendSignal(SIGINT, allowedTimeToExitNS: 10_000_000_000) + ], + logger: self.logger + ) - group.addTask { - let pidString = try await String(buffer: p.standardOutput.pullAllOfIt()) - guard let pid = pid_t(pidString.dropLast()) else { - XCTFail("couldn't get pid from \(pidString)") + try await withThrowingTaskGroup(of: pid_t?.self) { group in + group.addTask { + let result = try await p.run() + XCTAssertEqual(.exit(128 + SIGINT), result) return nil } - return pid - } - let maybePid = try await group.next()! - let pid = try XCTUnwrap(maybePid) - group.cancelAll() - try await group.waitForAll() + group.addTask { + let pidString = try await String(buffer: p.standardOutput.pullAllOfIt()) + guard let pid = pid_t(pidString.dropLast()) else { + XCTFail("couldn't get pid from \(pidString)") + return nil + } + return pid + } - // Let's check that the subprocess (/usr/bin/yes) of our subprocess (/bin/bash) is actually dead - // This is a tiny bit racy because the pid isn't immediately invalidated, so let's allow a few failures - for attempt in 0 ..< .max { - let killRet = kill(pid, 0) - let errnoCode = errno - guard killRet == -1 || attempt > 5 else { - logger.error("kill didn't fail on attempt \(attempt), trying again...") - usleep(100_000) - continue + let maybePid = try await group.next()! + let pid = try XCTUnwrap(maybePid) + group.cancelAll() + try await group.waitForAll() + + // Let's check that the subprocess (/usr/bin/yes) of our subprocess (/bin/bash) is actually dead + // This is a tiny bit racy because the pid isn't immediately invalidated, so let's allow a few failures + for attempt in 0 ..< .max { + let killRet = kill(pid, 0) + let errnoCode = errno + guard killRet == -1 || attempt > 5 else { + logger.error("kill didn't fail on attempt \(attempt), trying again...") + usleep(100_000) + continue + } + XCTAssertEqual(-1, killRet) + XCTAssertEqual(ESRCH, errnoCode) + break } - XCTAssertEqual(-1, killRet) - XCTAssertEqual(ESRCH, errnoCode) - break - } + } } - } #endif func testShutdownSequenceWorks() async throws { @@ -1115,8 +1121,8 @@ final class IntegrationTests: XCTestCase { group.addTask { let result = try await p.run() #if os(macOS) - // won't work on SCLF: https://github.com/apple/swift-corelibs-foundation/issues/4772 - XCTAssertEqual(.exit(3), result) + // won't work on SCLF: https://github.com/apple/swift-corelibs-foundation/issues/4772 + XCTAssertEqual(.exit(3), result) #endif } var allLines: [String] = [] @@ -1128,8 +1134,8 @@ final class IntegrationTests: XCTestCase { } try await group.waitForAll() #if os(macOS) - // won't work on SCLF: https://github.com/apple/swift-corelibs-foundation/issues/4772 - XCTAssertEqual(["OK", "saw SIGQUIT", "saw SIGTERM", "saw SIGINT"], allLines) + // won't work on SCLF: https://github.com/apple/swift-corelibs-foundation/issues/4772 + XCTAssertEqual(["OK", "saw SIGQUIT", "saw SIGTERM", "saw SIGINT"], allLines) #endif } } diff --git a/Tests/GeneratorEngineTests/EngineTests.swift b/Tests/GeneratorEngineTests/EngineTests.swift index 2e1a90b..5a3a95d 100644 --- a/Tests/GeneratorEngineTests/EngineTests.swift +++ b/Tests/GeneratorEngineTests/EngineTests.swift @@ -10,18 +10,22 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.Data import Crypto -@testable import Helpers +import XCTest + +import struct Foundation.Data import struct Logging.Logger import struct SystemPackage.FilePath -import XCTest + +@testable import Helpers private let encoder = JSONEncoder() private let decoder = JSONDecoder() -private extension AsyncFileSystem { - func read(_ path: FilePath, bufferLimit: Int = 10 * 1024 * 1024, as: V.Type) async throws -> V { +extension AsyncFileSystem { + fileprivate func read( + _ path: FilePath, bufferLimit: Int = 10 * 1024 * 1024, as: V.Type + ) async throws -> V { let data = try await self.withOpenReadableFile(path) { var data = Data() for try await chunk in try await $0.read() { @@ -35,7 +39,7 @@ private extension AsyncFileSystem { return try decoder.decode(V.self, from: data) } - func write(_ path: FilePath, _ value: some Encodable) async throws { + fileprivate func write(_ path: FilePath, _ value: some Encodable) async throws { let data = try encoder.encode(value) try await self.withOpenWritableFile(path) { fileHandle in try await fileHandle.write(data) @@ -116,7 +120,7 @@ final class EngineTests: XCTestCase { let engine = QueryEngine( MockFileSystem(), Logger(label: "engine-tests") -// cacheLocation: .memory + // cacheLocation: .memory ) var resultPath = try await engine[Expression(x: 1, y: 2)].path @@ -172,9 +176,11 @@ final class EngineTests: XCTestCase { func testQueryEncoding() throws { let item = MyItem( remoteURL: URL( - string: "https://download.swift.org/swift-5.9.2-release/ubuntu2204-aarch64/swift-5.9.2-RELEASE/swift-5.9.2-RELEASE-ubuntu22.04-aarch64.tar.gz" + string: + "https://download.swift.org/swift-5.9.2-release/ubuntu2204-aarch64/swift-5.9.2-RELEASE/swift-5.9.2-RELEASE-ubuntu22.04-aarch64.tar.gz" )!, - localPath: "/Users/katei/ghq/github.com/apple/swift-sdk-generator/Artifacts/target_swift_5.9.2-RELEASE_aarch64-unknown-linux-gnu.tar.gz", + localPath: + "/Users/katei/ghq/github.com/apple/swift-sdk-generator/Artifacts/target_swift_5.9.2-RELEASE_aarch64-unknown-linux-gnu.tar.gz", isPrebuilt: true ) func hashValue(of key: some CacheKey) throws -> SHA256Digest { diff --git a/Tests/HelpersTests/ThrowingDeferTests.swift b/Tests/HelpersTests/ThrowingDeferTests.swift index 2dc530e..03a0d3f 100644 --- a/Tests/HelpersTests/ThrowingDeferTests.swift +++ b/Tests/HelpersTests/ThrowingDeferTests.swift @@ -56,11 +56,13 @@ final class ThrowingDeferTests: XCTestCase { let workError = EquatableError() var didRunCleanup = false - XCTAssertThrowsError(try withThrowing { - throw workError - } defer: { - didRunCleanup = true - }) { + XCTAssertThrowsError( + try withThrowing { + throw workError + } defer: { + didRunCleanup = true + } + ) { XCTAssertTrue($0 is EquatableError) XCTAssertEqual($0 as? EquatableError, workError) } @@ -71,11 +73,13 @@ final class ThrowingDeferTests: XCTestCase { var didRunWork = false let cleanupError = EquatableError() - XCTAssertThrowsError(try withThrowing { - didRunWork = true - } defer: { - throw cleanupError - }) { + XCTAssertThrowsError( + try withThrowing { + didRunWork = true + } defer: { + throw cleanupError + } + ) { XCTAssertTrue($0 is EquatableError) XCTAssertEqual($0 as? EquatableError, cleanupError) } @@ -87,12 +91,14 @@ final class ThrowingDeferTests: XCTestCase { let workError = EquatableError() let cleanupError = EquatableError() - XCTAssertThrowsError(try withThrowing { - didRunWork = true - throw workError - } defer: { - throw cleanupError - }) { + XCTAssertThrowsError( + try withThrowing { + didRunWork = true + throw workError + } defer: { + throw cleanupError + } + ) { XCTAssertTrue($0 is EquatableError) XCTAssertEqual($0 as? EquatableError, cleanupError) } diff --git a/Tests/MacrosTests/MacrosTests.swift b/Tests/MacrosTests/MacrosTests.swift index f9a0b46..c8c457f 100644 --- a/Tests/MacrosTests/MacrosTests.swift +++ b/Tests/MacrosTests/MacrosTests.swift @@ -1,6 +1,6 @@ //===----------------------------------------------------------------------===// // -// This source file is part of the Swift.org open source project +// This source file is part of the Swift open source project // // Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception @@ -16,7 +16,9 @@ import SwiftSyntaxMacrosTestSupport import XCTest final class MacrosTests: XCTestCase { - private let macros: [String: Macro.Type] = ["CacheKey": CacheKeyMacro.self, "Query": QueryMacro.self] + private let macros: [String: Macro.Type] = [ + "CacheKey": CacheKeyMacro.self, "Query": QueryMacro.self, + ] func testCacheKeyDerived() { assertMacroExpansion( @@ -34,34 +36,34 @@ final class MacrosTests: XCTestCase { } """, expandedSource: """ - struct Message { - let text: String - let sender: String - } - struct Q { - let number: Int - let text: String - } + struct Message { + let text: String + let sender: String + } + struct Q { + let number: Int + let text: String + } - extension Message: CacheKeyProtocol { - func hash(with hashFunction: inout some HashFunction) { - String(reflecting: Self.self).hash(with: &hashFunction) - text.hash(with: &hashFunction) - sender.hash(with: &hashFunction) + extension Message: CacheKeyProtocol { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + text.hash(with: &hashFunction) + sender.hash(with: &hashFunction) + } } - } - extension Q: QueryProtocol { - } + extension Q: QueryProtocol { + } - extension Q: CacheKeyProtocol { - func hash(with hashFunction: inout some HashFunction) { - String(reflecting: Self.self).hash(with: &hashFunction) - number.hash(with: &hashFunction) - text.hash(with: &hashFunction) + extension Q: CacheKeyProtocol { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + number.hash(with: &hashFunction) + text.hash(with: &hashFunction) + } } - } - """, + """, macros: self.macros, indentationWidth: .spaces(2) ) diff --git a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift index 9202c2a..893d8c7 100644 --- a/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift +++ b/Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift @@ -11,9 +11,10 @@ //===----------------------------------------------------------------------===// import Logging -@testable import SwiftSDKGenerator import XCTest +@testable import SwiftSDKGenerator + final class ArchitectureMappingTests: XCTestCase { let logger = Logger(label: "ArchitectureMappingTests") @@ -39,12 +40,12 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple, targetTriple: Triple, - artifactID: String, // Base name of the generated bundle - hostLLVMDownloadURL: String, // URL of the host LLVM package - targetSwiftDownloadURL: String, // URL of the target Swift SDK + artifactID: String, // Base name of the generated bundle + hostLLVMDownloadURL: String, // URL of the host LLVM package + targetSwiftDownloadURL: String, // URL of the target Swift SDK - artifactBundlePathSuffix: String, // Path to the generated bundle - sdkDirPathSuffix: String // Path of the SDK within the bundle + artifactBundlePathSuffix: String, // Path to the generated bundle + sdkDirPathSuffix: String // Path of the SDK within the bundle ) async throws { let recipe = try LinuxRecipe( targetTriple: targetTriple, @@ -130,8 +131,10 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple("x86_64-apple-macosx13"), targetTriple: Triple("x86_64-unknown-linux-gnu"), artifactID: "5.8-RELEASE_ubuntu_jammy_x86_64", - hostLLVMDownloadURL: "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", - targetSwiftDownloadURL: "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", + hostLLVMDownloadURL: + "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", + targetSwiftDownloadURL: + "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_x86_64.artifactbundle", sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_x86_64/x86_64-unknown-linux-gnu/ubuntu-jammy.sdk" ) @@ -143,10 +146,13 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple("x86_64-apple-macosx13"), targetTriple: Triple("aarch64-unknown-linux-gnu"), artifactID: "5.8-RELEASE_ubuntu_jammy_aarch64", - hostLLVMDownloadURL: "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", - targetSwiftDownloadURL: "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", + hostLLVMDownloadURL: + "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", + targetSwiftDownloadURL: + "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_aarch64.artifactbundle", - sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" + sdkDirPathSuffix: + "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" ) } @@ -156,10 +162,13 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple("arm64-apple-macosx13"), targetTriple: Triple("aarch64-unknown-linux-gnu"), artifactID: "5.8-RELEASE_ubuntu_jammy_aarch64", - hostLLVMDownloadURL: "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", - targetSwiftDownloadURL: "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", + hostLLVMDownloadURL: + "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", + targetSwiftDownloadURL: + "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_aarch64.artifactbundle", - sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" + sdkDirPathSuffix: + "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" ) } @@ -169,8 +178,10 @@ final class ArchitectureMappingTests: XCTestCase { hostTriple: Triple("arm64-apple-macosx13"), targetTriple: Triple("x86_64-unknown-linux-gnu"), artifactID: "5.8-RELEASE_ubuntu_jammy_x86_64", - hostLLVMDownloadURL: "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", - targetSwiftDownloadURL: "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", + hostLLVMDownloadURL: + "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", + targetSwiftDownloadURL: + "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_x86_64.artifactbundle", sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_x86_64/x86_64-unknown-linux-gnu/ubuntu-jammy.sdk" ) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 15b19c9..e5abd83 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -18,7 +18,9 @@ import XCTest @testable import SwiftSDKGenerator extension FileManager { - func withTemporaryDirectory(logger: Logger, cleanup: Bool = true, body: (URL) async throws -> T) async throws -> T { + func withTemporaryDirectory( + logger: Logger, cleanup: Bool = true, body: (URL) async throws -> T + ) async throws -> T { // Create a temporary directory using a UUID. Throws if the directory already exists. // The docs suggest using FileManager.url(for: .itemReplacementDirectory, ...) to create a temporary directory, // but on Linux the directory name contains spaces, which means we need to be careful to quote it everywhere: @@ -33,15 +35,15 @@ extension FileManager { try createDirectory(at: temporaryDirectory, withIntermediateDirectories: false) defer { - // Best effort cleanup. - do { - if cleanup { - try removeItem(at: temporaryDirectory) - logger.info("Removed temporary directory") - } else { - logger.info("Keeping temporary directory") - } - } catch {} + // Best effort cleanup. + do { + if cleanup { + try removeItem(at: temporaryDirectory) + logger.info("Removed temporary directory") + } else { + logger.info("Keeping temporary directory") + } + } catch {} } logger.info("Created temporary directory") @@ -53,7 +55,9 @@ extension FileManager { // This takes a lock on `.build`, but if the tests are being run by `swift test` the outer Swift Package Manager // instance will already hold this lock, causing the test to deadlock. We can work around this by giving // the `swift run swift-sdk-generator` instance its own scratch directory. -func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: String) async throws -> String { +func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: String) + async throws -> String +{ var logger = logger logger[metadataKey: "runArguments"] = "\"\(runArguments)\"" logger[metadataKey: "scratchPath"] = "\(scratchPath)" @@ -69,9 +73,10 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: ) logger.info("Finished building SDK") - let installCommand = try XCTUnwrap(generatorOutput.split(separator: "\n").first { - $0.contains("swift experimental-sdk install") - }) + let installCommand = try XCTUnwrap( + generatorOutput.split(separator: "\n").first { + $0.contains("swift experimental-sdk install") + }) let bundleName = try XCTUnwrap( FilePath(String(XCTUnwrap(installCommand.split(separator: " ").last))).components.last @@ -79,7 +84,8 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: logger[metadataKey: "bundleName"] = "\(bundleName)" logger.info("Checking installed SDKs") - let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components(separatedBy: "\n") + let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components( + separatedBy: "\n") // Make sure this bundle hasn't been installed already. if installedSDKs.contains(bundleName) { @@ -116,7 +122,9 @@ final class RepeatedBuildTests: XCTestCase { func testRepeatedSDKBuilds() async throws { if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { - throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") + throw XCTSkip( + "EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145" + ) } var logger = logger @@ -127,15 +135,19 @@ final class RepeatedBuildTests: XCTestCase { var possibleArguments = ["--host-toolchain"] do { try await Shell.run("docker ps") - possibleArguments.append("--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9") + possibleArguments.append( + "--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9") } catch { - self.logger.warning("Docker CLI does not seem to be working, skipping tests that involve Docker.") + self.logger.warning( + "Docker CLI does not seem to be working, skipping tests that involve Docker.") } for runArguments in possibleArguments { if runArguments.contains("rhel") { // Temporarily skip the RHEL-based SDK. XCTSkip() is not suitable as it would skipping the entire test case - logger.warning("RHEL-based SDKs currently do not work with Swift 6.0: https://github.com/swiftlang/swift-sdk-generator/issues/138") + logger.warning( + "RHEL-based SDKs currently do not work with Swift 6.0: https://github.com/swiftlang/swift-sdk-generator/issues/138" + ) continue } @@ -155,7 +167,9 @@ struct SDKConfiguration { var architecture: String var withDocker: Bool - var bundleName: String { "\(linuxDistributionName)_\(linuxDistributionVersion)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" } + var bundleName: String { + "\(linuxDistributionName)_\(linuxDistributionVersion)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" + } func withDocker(_ enabled: Bool = true) -> SDKConfiguration { var res = self @@ -182,8 +196,8 @@ struct SDKConfiguration { "--swift-version \(swiftVersion)-RELEASE", testLinuxSwiftSDKs ? "--host \(hostArch!)-unknown-linux-gnu" : nil, "--target \(architecture)-unknown-linux-gnu", - "--linux-distribution-name \(linuxDistributionName)" - ].compactMap{ $0 }.joined(separator: " ") + "--linux-distribution-name \(linuxDistributionName)", + ].compactMap { $0 }.joined(separator: " ") } } @@ -199,10 +213,13 @@ var testLinuxSwiftSDKs: Bool { ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_TEST_LINUX_SWIFT_SDKS") } -func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) async throws { +func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) + async throws +{ let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") let testPackageDir = FilePath(testPackageURL.path) - try FileManager.default.createDirectory(atPath: testPackageDir.string, withIntermediateDirectories: true) + try FileManager.default.createDirectory( + atPath: testPackageDir.string, withIntermediateDirectories: true) logger.info("Creating test project \(testPackageDir)") try await Shell.run("swift package --package-path \(testPackageDir) init --type executable") @@ -227,7 +244,9 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD // that contains each Swift-supported Linux distribution. This way we can validate that each // distribution is capable of building using the Linux Swift SDK. if testLinuxSwiftSDKs { - let swiftContainerVersions = ["focal", "jammy", "noble", "fedora39", "rhel-ubi9", "amazonlinux2", "bookworm"] + let swiftContainerVersions = [ + "focal", "jammy", "noble", "fedora39", "rhel-ubi9", "amazonlinux2", "bookworm", + ] for containerVersion in swiftContainerVersions { logger.info("Building test project in 6.0-\(containerVersion) container") buildOutput = try await Shell.readStdout( @@ -241,7 +260,8 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD XCTAssertTrue(buildOutput.contains("Build complete!")) logger.info("Test project built successfully") - logger.info("Building test project in 6.0-\(containerVersion) container with static-swift-stdlib") + logger.info( + "Building test project in 6.0-\(containerVersion) container with static-swift-stdlib") buildOutput = try await Shell.readStdout( """ docker run --rm -v \(testPackageDir):/src \ @@ -277,7 +297,9 @@ func buildTestcases(config: SDKConfiguration) async throws { logger[metadataKey: "testcase"] = "testPackageInitExecutable" if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { - throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") + throw XCTSkip( + "EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145" + ) } if config.withDocker { @@ -289,7 +311,8 @@ func buildTestcases(config: SDKConfiguration) async throws { } let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - try await buildSDK(logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) + try await buildSDK( + logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) } logger.info("Built SDK") diff --git a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift index 9baf7de..b599075 100644 --- a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift +++ b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift @@ -35,7 +35,7 @@ final class SwiftSDKGeneratorMetadataTests: XCTestCase { bundleVersion: "0.0.3", targetTriple: Triple("armv7-unknown-linux-gnueabihf"), expectedCanonicalName: "armv7-swift-linux-gnueabihf" - ) + ), ] for testCase in testCases { diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 3c21f0d..656600e 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -27,7 +27,7 @@ final class LinuxRecipeTests: XCTestCase { targetSwiftPackagePath: String? = nil, includeHostToolchain: Bool = true ) throws -> LinuxRecipe { - try LinuxRecipe( + try LinuxRecipe( targetTriple: Triple("aarch64-unknown-linux-gnu"), hostTriple: hostTriple, linuxDistribution: .init(name: .ubuntu, version: "22.04"), @@ -48,7 +48,7 @@ final class LinuxRecipeTests: XCTestCase { targetTriple: Triple("x86_64-unknown-linux-gnu"), expectedSwiftCompilerOptions: [ "-Xlinker", "-R/usr/lib/swift/linux/", - "-Xclang-linker", "--ld-path=ld.lld" + "-Xclang-linker", "--ld-path=ld.lld", ], expectedLinkerPath: nil ), @@ -57,7 +57,7 @@ final class LinuxRecipeTests: XCTestCase { targetTriple: Triple("aarch64-unknown-linux-gnu"), expectedSwiftCompilerOptions: [ "-Xlinker", "-R/usr/lib/swift/linux/", - "-use-ld=lld" + "-use-ld=lld", ], expectedLinkerPath: "ld.lld" ), @@ -67,10 +67,10 @@ final class LinuxRecipeTests: XCTestCase { expectedSwiftCompilerOptions: [ "-Xlinker", "-R/usr/lib/swift/linux/", "-use-ld=lld", - "-latomic" + "-latomic", ], expectedLinkerPath: "ld.lld" - ) + ), ] for testCase in testCases { @@ -93,10 +93,12 @@ final class LinuxRecipeTests: XCTestCase { toolset: &toolset, targetTriple: Triple("x86_64-unknown-linux-gnu") ) XCTAssertEqual(toolset.rootPath, nil) - XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, [ - "-Xlinker", "-R/usr/lib/swift/linux/", - "-use-ld=lld" - ]) + XCTAssertEqual( + toolset.swiftCompiler?.extraCLIOptions, + [ + "-Xlinker", "-R/usr/lib/swift/linux/", + "-use-ld=lld", + ]) XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ["-lstdc++"]) XCTAssertEqual(toolset.librarian?.path, "llvm-ar") XCTAssert(toolset.linker == nil) @@ -117,9 +119,15 @@ final class LinuxRecipeTests: XCTestCase { pathsConfiguration ) let itemsToDownload = recipe.itemsToDownload(from: downloadableArtifacts) - let foundHostLLVM = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL }) - let foundTargetSwift = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL }) - let foundHostSwift = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.hostSwift.remoteURL }) + let foundHostLLVM = itemsToDownload.contains(where: { + $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL + }) + let foundTargetSwift = itemsToDownload.contains(where: { + $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL + }) + let foundHostSwift = itemsToDownload.contains(where: { + $0.remoteURL == downloadableArtifacts.hostSwift.remoteURL + }) // If this is a Linux host, we do not download LLVM XCTAssertEqual(foundHostLLVM, includesHostLLVM) @@ -129,53 +137,58 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForMacOSHost() throws { let hostTriple = Triple("x86_64-apple-macos") - let testCases: [(recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, includesHostSwift: Bool)] = [ - ( - // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), - includesHostLLVM: true, - includesTargetSwift: true, - includesHostSwift: true - ), - ( - // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), - includesHostLLVM: false, - includesTargetSwift: true, - includesHostSwift: true - ), - ( - // Remote target tarball with preinstalled toolchain - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), - includesHostLLVM: false, - includesTargetSwift: true, - includesHostSwift: false - ), - ( - // Local packages with Swift < 6.0 - recipe: try createRecipe( - hostTriple: hostTriple, - swiftVersion: "5.10", - hostSwiftPackagePath: "/path/to/host/swift", - targetSwiftPackagePath: "/path/to/target/swift" + let testCases: + [( + recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, + includesHostSwift: Bool + )] = [ + ( + // Remote tarballs on Swift < 6.0 + recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + includesHostLLVM: true, + includesTargetSwift: true, + includesHostSwift: true ), - includesHostLLVM: true, - includesTargetSwift: false, - includesHostSwift: false - ), - ( - // Local packages with Swift >= 6.0 - recipe: try createRecipe( - hostTriple: hostTriple, - swiftVersion: "6.0", - hostSwiftPackagePath: "/path/to/host/swift", - targetSwiftPackagePath: "/path/to/target/swift" + ( + // Remote tarballs on Swift >= 6.0 + recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + includesHostLLVM: false, + includesTargetSwift: true, + includesHostSwift: true ), - includesHostLLVM: false, - includesTargetSwift: false, - includesHostSwift: false - ) - ] + ( + // Remote target tarball with preinstalled toolchain + recipe: try createRecipe( + hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), + includesHostLLVM: false, + includesTargetSwift: true, + includesHostSwift: false + ), + ( + // Local packages with Swift < 6.0 + recipe: try createRecipe( + hostTriple: hostTriple, + swiftVersion: "5.10", + hostSwiftPackagePath: "/path/to/host/swift", + targetSwiftPackagePath: "/path/to/target/swift" + ), + includesHostLLVM: true, + includesTargetSwift: false, + includesHostSwift: false + ), + ( + // Local packages with Swift >= 6.0 + recipe: try createRecipe( + hostTriple: hostTriple, + swiftVersion: "6.0", + hostSwiftPackagePath: "/path/to/host/swift", + targetSwiftPackagePath: "/path/to/target/swift" + ), + includesHostLLVM: false, + includesTargetSwift: false, + includesHostSwift: false + ), + ] for testCase in testCases { try runItemsToDownloadTestCase( @@ -204,7 +217,8 @@ final class LinuxRecipeTests: XCTestCase { ), ( // Remote target tarball with preinstalled toolchain - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), + recipe: try createRecipe( + hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), includesTargetSwift: true, includesHostSwift: false ), @@ -229,7 +243,7 @@ final class LinuxRecipeTests: XCTestCase { ), includesTargetSwift: false, includesHostSwift: false - ) + ), ] for testCase in testCases { @@ -253,7 +267,10 @@ final class LinuxRecipeTests: XCTestCase { (swiftVersion: "5.9", includeHostToolchain: false, expectedHostTriples: allHostTriples), (swiftVersion: "5.10", includeHostToolchain: false, expectedHostTriples: allHostTriples), (swiftVersion: "6.0", includeHostToolchain: false, expectedHostTriples: nil), - (swiftVersion: "6.0", includeHostToolchain: true, expectedHostTriples: [Triple("x86_64-unknown-linux-gnu")]) + ( + swiftVersion: "6.0", includeHostToolchain: true, + expectedHostTriples: [Triple("x86_64-unknown-linux-gnu")] + ), ] for testCase in testCases { diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift index 71eff42..080a13b 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -17,7 +17,7 @@ import XCTest final class WebAssemblyRecipeTests: XCTestCase { let logger = Logger(label: "WebAssemblyRecipeTests") - + func createRecipe() -> WebAssemblyRecipe { WebAssemblyRecipe( hostSwiftPackage: nil, @@ -65,8 +65,10 @@ final class WebAssemblyRecipeTests: XCTestCase { ] XCTAssertEqual(toolset.cCompiler?.extraCLIOptions, ccOptions) XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ccOptions) - XCTAssertEqual(toolset.linker?.extraCLIOptions, [ - "--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", - ]) + XCTAssertEqual( + toolset.linker?.extraCLIOptions, + [ + "--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", + ]) } } diff --git a/Utilities/soundness.sh b/Utilities/soundness.sh deleted file mode 100755 index 3ff517d..0000000 --- a/Utilities/soundness.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift open source project -## -## Copyright (c) 2022-2023 Apple Inc. and the Swift project authors -## Licensed under Apache License v2.0 with Runtime Library Exception -## -## See https://swift.org/LICENSE.txt for license information -## See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -## -##===----------------------------------------------------------------------===## - -set -eu -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -function replace_acceptable_years() { - # this needs to replace all acceptable forms with 'YEARS' - sed -e 's/20[12][0123456789]-20[12][0123456789]/YEARS/' -e 's/20[12][0123456789]/YEARS/' -} - -printf "=> Checking for unacceptable language... " -# This greps for unacceptable terminology. The square bracket[s] are so that -# "git grep" doesn't find the lines that greps :). -unacceptable_terms=( - -e blacklis[t] - -e whitelis[t] - -e slav[e] - -e sanit[y] - -e maste[r] -) - -# We have to exclude the code of conduct as it gives examples of unacceptable language. -if git grep --color=never -i "${unacceptable_terms[@]}" -- . > /dev/null; then - printf "\033[0;31mUnacceptable language found.\033[0m\n" - git grep -i "${unacceptable_terms[@]}" -- . - exit 1 -fi -printf "\033[0;32mokay.\033[0m\n" - -printf "=> Checking format... \n" -git diff --name-only | grep ".swift" | while read changed_file; do - printf " * checking ${changed_file}... " - before=$(cat "${changed_file}") - swiftformat "$changed_file" > /dev/null 2>&1 - after=$(cat "${changed_file}") - - if [[ "$before" != "$after" ]]; then - printf "\033[0;31mformatting issues!\033[0m\n" - git --no-pager diff "${changed_file}" - exit 1 - else - printf "\033[0;32mokay.\033[0m\n" - fi -done - -printf "=> Checking license headers... \n" -tmp=$(mktemp /tmp/.swift-package-manager-soundness_XXXXXX) - -for language in swift-or-c bash python; do - printf " * $language... " - declare -a matching_files - declare -a exceptions - expections=( ) - matching_files=( -name '*' ) - case "$language" in - swift-or-c) - exceptions=( -name "Package.swift" -o -path "./Examples/*" -o -path "./Fixtures/*" \ - -o -path "./IntegrationTests/*" -o -path "./Tests/*" -o -path "./Bundles/*" ) - matching_files=( -name '*.swift' -o -name '*.c' -o -name '*.h' ) - cat > "$tmp" <<"EOF" -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) YEARS Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -EOF - ;; - bash) - exceptions=( -path "./Examples/*" -o -path "./Fixtures/*" -o -path "./IntegrationTests/*" \ - -o -path "./Bundles/*" ) - matching_files=( -name '*.sh' ) - cat > "$tmp" <<"EOF" -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift open source project -## -## Copyright (c) YEARS Apple Inc. and the Swift project authors -## Licensed under Apache License v2.0 with Runtime Library Exception -## -## See https://swift.org/LICENSE.txt for license information -## See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -## -##===----------------------------------------------------------------------===## -EOF - ;; - python) - exceptions=( -path "./Examples/*" -o -path "./Fixtures/*" -o -path "./IntegrationTests/*" \ - -o -path "./Bundles/*" ) - matching_files=( -name '*.py' ) - cat > "$tmp" <<"EOF" -#!/usr/bin/env python3 -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift open source project -## -## Copyright (c) YEARS Apple Inc. and the Swift project authors -## Licensed under Apache License v2.0 with Runtime Library Exception -## -## See https://swift.org/LICENSE.txt for license information -## See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -## -##===----------------------------------------------------------------------===## -EOF - ;; - *) - echo >&2 "ERROR: unknown language '$language'" - ;; - esac - - expected_lines=$(cat "$tmp" | wc -l) - expected_sha=$(cat "$tmp" | shasum) - - ( - cd "$here/.." - { - find . \ - \( \! -path './.build/*' -a \ - \( "${matching_files[@]}" \) -a \ - \( \! \( "${exceptions[@]}" \) \) \) - - } | while read line; do - if [[ "$(cat "$line" | replace_acceptable_years | head -n $expected_lines | shasum)" != "$expected_sha" ]]; then - printf "\033[0;31mmissing headers in file '$line'!\033[0m\n" - diff -u <(cat "$line" | replace_acceptable_years | head -n $expected_lines) "$tmp" - exit 1 - fi - done - printf "\033[0;32mokay.\033[0m\n" - ) -done - -rm "$tmp" From 4de34cea3d907be80af959a37cb4971d0c32c253 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Mon, 10 Mar 2025 12:09:42 +0000 Subject: [PATCH 04/17] workflows: Run unit tests using GitHub Actions (#200) * workflows: Re-enable tests job * workflows: Disable tests on Windows Requires NIO support on Windows * workflows: Disable end-to-end tests in GitHub Actions There is a good chance that GitHub Actions will be able to run these tests, but we will disable them for now in order to get the unit tests running. The end-to-end tests could not run with swift-ci, so disabling them here does not reduce CI coverage. --- .github/workflows/pull_request.yml | 11 ++++++----- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 8 ++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c06cc36..fff448d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5,11 +5,12 @@ on: types: [opened, reopened, synchronize] jobs: - # tests: - # name: Test - # uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main - # with: - # linux_pre_build_command: apt-get update && apt-get install -y locales locales-all libsqlite3-dev + tests: + name: Test + uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main + with: + linux_pre_build_command: apt-get update && apt-get install -y locales locales-all libsqlite3-dev + enable_windows_checks: false soundness: name: Soundness uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index e5abd83..e18f543 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -121,7 +121,9 @@ final class RepeatedBuildTests: XCTestCase { private let logger = Logger(label: "swift-sdk-generator") func testRepeatedSDKBuilds() async throws { - if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { + if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") + || ProcessInfo.processInfo.environment.keys.contains("GITHUB_ACTIONS") + { throw XCTSkip( "EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145" ) @@ -296,7 +298,9 @@ func buildTestcases(config: SDKConfiguration) async throws { var logger = Logger(label: "EndToEndTests") logger[metadataKey: "testcase"] = "testPackageInitExecutable" - if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { + if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") + || ProcessInfo.processInfo.environment.keys.contains("GITHUB_ACTIONS") + { throw XCTSkip( "EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145" ) From 330c3f27e699174b74cf4a77777974e41acfc333 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 12 Mar 2025 11:19:56 -0400 Subject: [PATCH 05/17] Fix support for Amazon Linux 2 x86_64, add EndToEndTests for all RHEL distributions (#195) This fixes the final issue we saw with #138 where there was an error generating an `amazonlinux2` Swift SDK for x86_64. This was fixed by removing 32-bit libraries from the sysroot imported from the container for RHEL-based distributions. To test this, I've added EndToEndTests for `amazonlinux2` and `fedora39` on top of the existing RHEL UBI9 tests. All of these depend on pulling from containers to test: ``` 2.1 GiB [#################] /fedora39_aarch64_6.0.3-RELEASE_with-docker.artifactbundle 2.0 GiB [################ ] /fedora39_x86_64_6.0.3-RELEASE_with-docker.artifactbundle 2.0 GiB [################ ] /amazonlinux2_aarch64_6.0.3-RELEASE_with-docker.artifactbundle 1.9 GiB [############### ] /amazonlinux2_x86_64_6.0.3-RELEASE_with-docker.artifactbundle 1.9 GiB [############### ] /rhel_ubi9_x86_64_6.0.3-RELEASE_with-docker.artifactbundle 1.9 GiB [############### ] /rhel_ubi9_aarch64_6.0.3-RELEASE_with-docker.artifactbundle 1.8 GiB [############### ] /fedora39_x86_64_5.10.1-RELEASE_with-docker.artifactbundle 1.8 GiB [############### ] /fedora39_aarch64_5.10.1-RELEASE_with-docker.artifactbundle 1.8 GiB [############## ] /amazonlinux2_aarch64_5.10.1-RELEASE_with-docker.artifactbundle 1.8 GiB [############## ] /amazonlinux2_x86_64_5.10.1-RELEASE_with-docker.artifactbundle 1.7 GiB [############## ] /rhel_ubi9_x86_64_5.10.1-RELEASE_with-docker.artifactbundle 1.7 GiB [############## ] /rhel_ubi9_aarch64_5.10.1-RELEASE_with-docker.artifactbundle 1.7 GiB [############## ] /amazonlinux2_aarch64_5.9.2-RELEASE_with-docker.artifactbundle 1.7 GiB [############# ] /amazonlinux2_x86_64_5.9.2-RELEASE_with-docker.artifactbundle 1.7 GiB [############# ] /rhel_ubi9_x86_64_5.9.2-RELEASE_with-docker.artifactbundle 1.7 GiB [############# ] /rhel_ubi9_aarch64_5.9.2-RELEASE_with-docker.artifactbundle ``` All of these Swift SDKs take 30GB of disk space. --------- Co-authored-by: Max Desiatov --- .../Generator/SwiftSDKGenerator+Copy.swift | 15 ++ .../EndToEndTests.swift | 137 ++++++++++++++---- 2 files changed, 126 insertions(+), 26 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index e561cd2..e1386ac 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -91,6 +91,21 @@ extension SwiftSDKGenerator { } try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib") + // Look for 32-bit libraries to remove from RHEL-based distros + // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up + if case .rhel = targetDistribution { + for gccVersion in 7...13 { + let removePath = "gcc/x86_64-redhat-linux/\(gccVersion)/32" + if await doesFileExist(at: sdkUsrLibPath.appending(removePath)) { + logger.warning( + "Removing 32-bit libraries from RHEL imported sysroot", + metadata: ["removePath": .stringConvertible(removePath)] + ) + try await removeRecursively(at: sdkUsrLibPath.appending(removePath)) + } + } + } + // Copy the ELF interpreter try await generator.copyFromDockerContainer( id: containerID, diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index e18f543..b067fd7 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -36,14 +36,12 @@ extension FileManager { try createDirectory(at: temporaryDirectory, withIntermediateDirectories: false) defer { // Best effort cleanup. - do { - if cleanup { - try removeItem(at: temporaryDirectory) - logger.info("Removed temporary directory") - } else { - logger.info("Keeping temporary directory") - } - } catch {} + if cleanup { + try? removeItem(at: temporaryDirectory) + logger.info("Removed temporary directory") + } else { + logger.info("Keeping temporary directory") + } } logger.info("Created temporary directory") @@ -62,7 +60,7 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: logger[metadataKey: "runArguments"] = "\"\(runArguments)\"" logger[metadataKey: "scratchPath"] = "\(scratchPath)" - logger.info("Building SDK") + logger.info("Building Swift SDK") var packageDirectory = FilePath(#filePath) packageDirectory.removeLastComponent() @@ -71,7 +69,7 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: let generatorOutput = try await Shell.readStdout( "cd \(packageDirectory) && swift run --scratch-path \"\(scratchPath)\" swift-sdk-generator make-linux-sdk \(runArguments)" ) - logger.info("Finished building SDK") + logger.info("Finished building Swift SDK") let installCommand = try XCTUnwrap( generatorOutput.split(separator: "\n").first { @@ -83,17 +81,17 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: ).stem logger[metadataKey: "bundleName"] = "\(bundleName)" - logger.info("Checking installed SDKs") + logger.info("Checking installed Swift SDKs") let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components( separatedBy: "\n") // Make sure this bundle hasn't been installed already. if installedSDKs.contains(bundleName) { - logger.info("Removing existing SDK") + logger.info("Removing existing Swift SDK") try await Shell.run("swift experimental-sdk remove \(bundleName)") } - logger.info("Installing new SDK") + logger.info("Installing new Swift SDK") let installOutput = try await Shell.readStdout(String(installCommand)) XCTAssertTrue(installOutput.contains("successfully installed")) @@ -168,9 +166,11 @@ struct SDKConfiguration { var linuxDistributionVersion: String var architecture: String var withDocker: Bool + var containerImageSuffix: String? var bundleName: String { - "\(linuxDistributionName)_\(linuxDistributionVersion)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" + let sdkPrefix = containerImageSuffix ?? "\(linuxDistributionName)_\(linuxDistributionVersion)" + return "\(sdkPrefix)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" } func withDocker(_ enabled: Bool = true) -> SDKConfiguration { @@ -179,6 +179,12 @@ struct SDKConfiguration { return res } + func withContainerImageSuffix(_ containerImageSuffix: String) -> SDKConfiguration { + var res = self + res.containerImageSuffix = containerImageSuffix + return res + } + func withArchitecture(_ arch: String) -> SDKConfiguration { var res = self res.architecture = arch @@ -191,14 +197,22 @@ struct SDKConfiguration { } var sdkGeneratorArguments: String { + // Build the container image tag + var containerImage: String? = nil + if let containerImageSuffix { + containerImage = "swift:\(swiftVersion)-\(containerImageSuffix)" + } + return [ "--sdk-name \(bundleName)", "--host-toolchain", withDocker ? "--with-docker" : nil, + containerImage != nil ? "--from-container-image" : nil, containerImage, "--swift-version \(swiftVersion)-RELEASE", testLinuxSwiftSDKs ? "--host \(hostArch!)-unknown-linux-gnu" : nil, "--target \(architecture)-unknown-linux-gnu", "--linux-distribution-name \(linuxDistributionName)", + "--linux-distribution-version \(linuxDistributionVersion)", ].compactMap { $0 }.joined(separator: " ") } } @@ -319,17 +333,28 @@ func buildTestcases(config: SDKConfiguration) async throws { logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) } - logger.info("Built SDK") + logger.info("Built Swift SDK") + + // Cleanup + func cleanupSDK() async { + logger.info("Removing Swift SDK to clean up...") + try? await Shell.run("swift experimental-sdk remove \(bundleName)") + } for testcase in testcases { - try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - try await buildTestcase(logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir) + do { + try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + try await buildTestcase( + logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir + ) + } + } catch { + await cleanupSDK() + throw error } } - // Cleanup - logger.info("Removing SDK to cleanup...") - try await Shell.run("swift experimental-sdk remove \(bundleName)") + await cleanupSDK() } final class Swift59_UbuntuEndToEndTests: XCTestCase { @@ -433,12 +458,24 @@ final class Swift59_RHELEndToEndTests: XCTestCase { func testAarch64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + try await buildTestcases(config: config.withArchitecture("aarch64")) } func testX86_64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2")) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2")) } } @@ -453,12 +490,36 @@ final class Swift510_RHELEndToEndTests: XCTestCase { func testAarch64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + try await buildTestcases(config: config.withArchitecture("aarch64")) } func testX86_64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2")) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2")) + } + + func testFedora39Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39")) + } + + func testFedora39X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39")) } } @@ -473,11 +534,35 @@ final class Swift60_RHELEndToEndTests: XCTestCase { func testAarch64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + try await buildTestcases(config: config.withArchitecture("aarch64")) } func testX86_64FromContainer() async throws { try skipSlow() - try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2")) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2")) + } + + func testFedora39Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39")) + } + + func testFedora39X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39")) } } From 685190d85b0f460b87cf8532c4ddba381a293ebd Mon Sep 17 00:00:00 2001 From: xavgru12 <117173294+xavgru12@users.noreply.github.com> Date: Fri, 21 Mar 2025 17:56:58 +0100 Subject: [PATCH 06/17] Fix typo in `README.md` (#202) `mkdir cross-compilation test` -> `mkdir cross-compilation-test` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d456d4d..5ec9617 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ subsequently as ``. Create a new project to verify that the SDK works: ``` -mkdir cross-compilation test +mkdir cross-compilation-test cd cross-compilation-test swift package init --type executable ``` From 0158a412b5a7d99457f51e843bbf8dad584849a3 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 3 Apr 2025 17:17:29 +0100 Subject: [PATCH 07/17] Set `lineBreakBeforeEachArgument` to `true` in `.swift-format` (#205) Make formatting consistent and closer to the widely adopted style. --- .swift-format | 2 +- Sources/AsyncProcess/FileContentStream.swift | 7 ++- .../ProcessExecutor+Convenience.swift | 8 ++- Sources/AsyncProcess/ProcessExecutor.swift | 8 ++- Sources/GeneratorCLI/GeneratorCLI.swift | 34 +++++++--- .../Generator/SwiftSDKGenerator+Copy.swift | 3 +- .../SwiftSDKGenerator+Download.swift | 26 +++++--- .../SwiftSDKGenerator+Entrypoint.swift | 5 +- .../Generator/SwiftSDKGenerator+Fixup.swift | 3 +- .../SwiftSDKGenerator+Metadata.swift | 3 +- .../Generator/SwiftSDKGenerator+Unpack.swift | 3 +- .../Generator/SwiftSDKGenerator.swift | 13 ++-- .../PlatformModels/LinuxDistribution.swift | 4 +- .../Queries/DownloadArtifactQuery.swift | 9 ++- .../SwiftSDKRecipes/LinuxRecipe.swift | 16 +++-- .../SwiftSDKRecipes/SwiftSDKRecipe.swift | 7 ++- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 27 +++++--- .../SystemUtils/ByteBuffer+Utils.swift | 3 +- .../SystemUtils/HTTPClient+Download.swift | 27 +++++--- .../SwiftSDKGenerator/SystemUtils/which.swift | 4 +- .../AsyncByteBufferLineSequenceTests.swift | 3 +- .../AsyncProcessTests/IntegrationTests.swift | 40 ++++++++---- Tests/GeneratorEngineTests/EngineTests.swift | 4 +- .../EndToEndTests.swift | 63 +++++++++++++------ .../SwiftSDKRecipes/LinuxRecipeTests.swift | 30 ++++++--- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 9 ++- 26 files changed, 255 insertions(+), 106 deletions(-) diff --git a/.swift-format b/.swift-format index e345c2e..3b4cc63 100644 --- a/.swift-format +++ b/.swift-format @@ -9,7 +9,7 @@ }, "lineBreakAroundMultilineExpressionChainComponents" : false, "lineBreakBeforeControlFlowKeywords" : false, - "lineBreakBeforeEachArgument" : false, + "lineBreakBeforeEachArgument" : true, "lineBreakBeforeEachGenericRequirement" : false, "lineLength" : 100, "maximumBlankLines" : 1, diff --git a/Sources/AsyncProcess/FileContentStream.swift b/Sources/AsyncProcess/FileContentStream.swift index c0ee126..4999b8f 100644 --- a/Sources/AsyncProcess/FileContentStream.swift +++ b/Sources/AsyncProcess/FileContentStream.swift @@ -256,7 +256,9 @@ private final class ReadIntoAsyncChannelHandler: ChannelDuplexHandler { extension FileHandle { func fileContentStream(eventLoop: EventLoop) throws -> FileContentStream { let asyncBytes = try FileContentStream( - fileDescriptor: self.fileDescriptor, eventLoop: eventLoop) + fileDescriptor: self.fileDescriptor, + eventLoop: eventLoop + ) try self.close() return asyncBytes } @@ -432,7 +434,8 @@ where Base: AsyncSequence, Base.Element == ByteBuffer { } public init( - _ underlying: Base, dropTerminator: Bool, + _ underlying: Base, + dropTerminator: Bool, maximumAllowableBufferSize: Int, dropLastChunkIfNoNewline: Bool ) { diff --git a/Sources/AsyncProcess/ProcessExecutor+Convenience.swift b/Sources/AsyncProcess/ProcessExecutor+Convenience.swift index 00c6607..59e0302 100644 --- a/Sources/AsyncProcess/ProcessExecutor+Convenience.swift +++ b/Sources/AsyncProcess/ProcessExecutor+Convenience.swift @@ -324,7 +324,10 @@ extension ProcessExecutor { } var allInfo = ProcessExitReasonAndOutput( - exitReason: .exit(-1), standardOutput: nil, standardError: nil) + exitReason: .exit(-1), + standardOutput: nil, + standardError: nil + ) while let next = try await group.next() { switch next { case let .exitReason(exitReason): @@ -467,7 +470,8 @@ extension ProcessExecutor { try await self.runCollectingOutput( group: group, executable: executable, - arguments, standardInput: EOFSequence(), + arguments, + standardInput: EOFSequence(), collectStandardOutput: collectStandardOutput, collectStandardError: collectStandardError, perStreamCollectionLimitBytes: perStreamCollectionLimitBytes, diff --git a/Sources/AsyncProcess/ProcessExecutor.swift b/Sources/AsyncProcess/ProcessExecutor.swift index 80429a6..4fe211d 100644 --- a/Sources/AsyncProcess/ProcessExecutor.swift +++ b/Sources/AsyncProcess/ProcessExecutor.swift @@ -159,7 +159,8 @@ public final actor ProcessExecutor { private let _standardOutput: ChunkSequence private let _standardError: ChunkSequence private let processIsRunningApproximation = ManagedAtomic( - RunningStateApproximation.neverStarted.rawValue) + RunningStateApproximation.neverStarted.rawValue + ) private let processOutputConsumptionApproximation = ManagedAtomic(UInt8(0)) private let processPid = ManagedAtomic(pid_t(0)) private let ownsStandardOutputWriteHandle: Bool @@ -568,7 +569,10 @@ public final actor ProcessExecutor { // At this point, the process is running, we should therefore have a process ID (unless we're already dead). let childPid = p.processIdentifier _ = self.processPid.compareExchange( - expected: 0, desired: childPid, ordering: .sequentiallyConsistent) + expected: 0, + desired: childPid, + ordering: .sequentiallyConsistent + ) assert(childPid != 0 || !p.isRunning) self.logger.debug( "running command", diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index 82667af..cef47dc 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -71,7 +71,8 @@ struct GeneratorCLI: AsyncParsableCommand { logger.info( "Generator run finished successfully.", - metadata: ["elapsedTime": .string(elapsed.intervalString)]) + metadata: ["elapsedTime": .string(elapsed.intervalString)] + ) } } @@ -167,7 +168,8 @@ extension GeneratorCLI { if let arch = hostArch { let target = Triple(arch: arch, vendor: current.vendor!, os: current.os!) appLogger.warning( - "deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`") + "deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`" + ) return target } return current @@ -219,7 +221,8 @@ extension GeneratorCLI { if let arch = generatorOptions.targetArch { let target = Triple(arch: arch, vendor: nil, os: .linux, environment: .gnu) appLogger.warning( - "deprecated: Please use `--target \(target.triple)` instead of `--target-arch \(arch)`") + "deprecated: Please use `--target \(target.triple)` instead of `--target-arch \(arch)`" + ) } return Triple(arch: hostTriple.arch!, vendor: nil, os: .linux, environment: .gnu) } @@ -240,7 +243,9 @@ extension GeneratorCLI { let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion let linuxDistribution = try LinuxDistribution( - name: linuxDistributionName, version: linuxDistributionVersion) + name: linuxDistributionName, + version: linuxDistributionVersion + ) let hostTriple = try self.generatorOptions.deriveHostTriple() let targetTriple = self.deriveTargetTriple(hostTriple: hostTriple) @@ -259,7 +264,10 @@ extension GeneratorCLI { logger: loggerWithLevel(from: self.generatorOptions) ) try await GeneratorCLI.run( - recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions) + recipe: recipe, + targetTriple: targetTriple, + options: self.generatorOptions + ) } func isInvokedAsDefaultSubcommand() -> Bool { @@ -316,7 +324,10 @@ extension GeneratorCLI { ) let targetTriple = self.deriveTargetTriple() try await GeneratorCLI.run( - recipe: recipe, targetTriple: targetTriple, options: self.generatorOptions) + recipe: recipe, + targetTriple: targetTriple, + options: self.generatorOptions + ) } } } @@ -327,12 +338,19 @@ extension Duration { let date = Date(timeInterval: TimeInterval(self.components.seconds), since: reference) let components = Calendar.current.dateComponents( - [.hour, .minute, .second], from: reference, to: date) + [.hour, .minute, .second], + from: reference, + to: date + ) if let hours = components.hour, hours > 0 { #if !canImport(Darwin) && compiler(<6.0) return String( - format: "%02d:%02d:%02d", hours, components.minute ?? 0, components.second ?? 0) + format: "%02d:%02d:%02d", + hours, + components.minute ?? 0, + components.second ?? 0 + ) #else return self.formatted() #endif diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index e1386ac..3942662 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -136,7 +136,8 @@ extension SwiftSDKGenerator { if isOptional && !doesFileExist(at: fromPath) { logger.debug( "Optional package path ignored since it does not exist", - metadata: ["packagePath": .string(fromPath.string)]) + metadata: ["packagePath": .string(fromPath.string)] + ) continue } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index ad84680..0807fe0 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -31,7 +31,8 @@ extension FilePath { extension SwiftSDKGenerator { func downloadArtifacts( - _ client: some HTTPClientProtocol, _ engine: QueryEngine, + _ client: some HTTPClientProtocol, + _ engine: QueryEngine, downloadableArtifacts: inout DownloadableArtifacts, itemsToDownload: @Sendable (DownloadableArtifacts) -> [DownloadableArtifacts.Item] ) async throws { @@ -58,7 +59,8 @@ extension SwiftSDKGenerator { for item in itemsToDownload(downloadableArtifacts) { group.addTask { try await engine[ - DownloadArtifactQuery(artifact: item, httpClient: client, logger: self.logger)] + DownloadArtifactQuery(artifact: item, httpClient: client, logger: self.logger) + ] } } @@ -74,7 +76,8 @@ extension SwiftSDKGenerator { "Using downloaded artifacts in these locations.", metadata: [ "paths": .array(results.map(\.path.metadataValue)) - ]) + ] + ) } func downloadUbuntuPackages( @@ -93,7 +96,8 @@ extension SwiftSDKGenerator { """ The `xz` utility was not found in `PATH`. \ Consider installing it for more efficient downloading of package lists. - """) + """ + ) } async let mainPackages = try await client.parseUbuntuPackagesList( @@ -137,7 +141,9 @@ extension SwiftSDKGenerator { } logger.info( - "Downloading Ubuntu packages...", metadata: ["packageCount": .stringConvertible(urls.count)]) + "Downloading Ubuntu packages...", + metadata: ["packageCount": .stringConvertible(urls.count)] + ) try await inTemporaryDirectory { fs, tmpDir in let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine) await report(downloadedFiles: downloadedFiles) @@ -160,8 +166,11 @@ extension SwiftSDKGenerator { $0.addTask { let downloadedFilePath = try await engine[ DownloadFileQuery( - remoteURL: url, localDirectory: directory, httpClient: client - )] + remoteURL: url, + localDirectory: directory, + httpClient: client + ) + ] let filePath = downloadedFilePath.path guard let fileSize = try FileManager.default.attributesOfItem( @@ -191,7 +200,8 @@ extension SwiftSDKGenerator { metadata: [ "url": .string(url.absoluteString), "size": .string(byteCountFormatter.string(fromByteCount: Int64(bytes))), - ]) + ] + ) } } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index fa95b6b..e5c9cf6 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -51,7 +51,10 @@ extension SwiftSDKGenerator { try await self.createDirectoryIfNeeded(at: pathsConfiguration.artifactsCachePath) let swiftSDKProduct = try await recipe.makeSwiftSDK( - generator: self, engine: engine, httpClient: client) + generator: self, + engine: engine, + httpClient: client + ) let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift index 3161b5e..493db05 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift @@ -58,7 +58,8 @@ extension SwiftSDKGenerator { func symlinkClangHeaders() throws { let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending( - "usr/lib/swift_static/clang") + "usr/lib/swift_static/clang" + ) if !doesFileExist(at: swiftStaticClangPath) { logger.info("Symlinking clang headers...") try self.createSymlink( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index 9a27475..e35ae01 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -63,7 +63,8 @@ extension SwiftSDKGenerator { """ `toolchainBinDirPath`, `sdkDirPath`, and `toolsetPath` are at unexpected locations that prevent computing \ relative paths - """) + """ + ) } var metadata = SwiftSDKMetadataV4.TripleProperties( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index 563936f..f4c953c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -103,7 +103,8 @@ extension SwiftSDKGenerator { try await inTemporaryDirectory { fs, tmpDir in try await fs.unpack(file: targetSwiftPackagePath, into: tmpDir) try await fs.copyTargetSwift( - from: tmpDir.appending(relativePathToRoot).appending("usr"), sdkDirPath: sdkDirPath + from: tmpDir.appending(relativePathToRoot).appending("usr"), + sdkDirPath: sdkDirPath ) } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index 06a9b27..b14b61c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -72,7 +72,8 @@ public actor SwiftSDKGenerator { return Triple("\(cpu)-unknown-linux-gnu") #else fatalError( - "Triple detection not implemented for the platform that this generator was built on.") + "Triple detection not implemented for the platform that this generator was built on." + ) #endif } @@ -195,7 +196,8 @@ public actor SwiftSDKGenerator { if isSymlink { let path = url.path try result.append( - (FilePath(path), FilePath(self.fileManager.destinationOfSymbolicLink(atPath: url.path)))) + (FilePath(path), FilePath(self.fileManager.destinationOfSymbolicLink(atPath: url.path))) + ) } } @@ -235,7 +237,9 @@ public actor SwiftSDKGenerator { func gunzip(file: FilePath, into directoryPath: FilePath) async throws { try await Shell.run( - #"cd "\#(directoryPath)" && gzip -d "\#(file)""#, shouldLogCommands: self.isVerbose) + #"cd "\#(directoryPath)" && gzip -d "\#(file)""#, + shouldLogCommands: self.isVerbose + ) } func untar( @@ -264,7 +268,8 @@ public actor SwiftSDKGenerator { let lsOutput = try await Shell.readStdout(cmd) logger.debug( "Files unpacked from deb file", - metadata: ["cmd": .string(cmd), "output": .string(lsOutput)]) + metadata: ["cmd": .string(cmd), "output": .string(lsOutput)] + ) } try await Shell.run( diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index 0152478..5385057 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -35,7 +35,9 @@ public enum LinuxDistribution: Hashable, Sendable { self = .noble default: throw GeneratorError.unknownLinuxDistribution( - name: LinuxDistribution.Name.ubuntu.rawValue, version: version) + name: LinuxDistribution.Name.ubuntu.rawValue, + version: version + ) } } diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift index e81d2c3..adc8eb5 100644 --- a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift @@ -25,9 +25,11 @@ struct DownloadArtifactQuery: Query { func run(engine: QueryEngine) async throws -> FilePath { logger.info( "Downloading remote artifact not available in local cache", - metadata: ["remoteUrl": .string(self.artifact.remoteURL.absoluteString)]) + metadata: ["remoteUrl": .string(self.artifact.remoteURL.absoluteString)] + ) let stream = self.httpClient.streamDownloadProgress( - from: self.artifact.remoteURL, to: self.artifact.localPath + from: self.artifact.remoteURL, + to: self.artifact.localPath ) .removeDuplicates(by: didProgressChangeSignificantly) ._throttle(for: .seconds(1)) @@ -51,7 +53,8 @@ struct DownloadArtifactQuery: Query { byteCountFormatter .string(fromByteCount: Int64(total)) ) - """) + """ + ) } else { logger.debug( "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))" diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 7d58357..923462d 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -225,7 +225,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) async throws -> SwiftSDKProduct { if self.linuxDistribution.name == .rhel && self.mainTargetTriple.archName == "armv7" { throw GeneratorError.distributionDoesNotSupportArchitecture( - self.linuxDistribution, targetArchName: self.mainTargetTriple.archName + self.linuxDistribution, + targetArchName: self.mainTargetTriple.archName ) } @@ -268,7 +269,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { switch self.hostSwiftSource { case let .localPackage(filePath): try await generator.rsync( - from: filePath.appending("usr"), to: generator.pathsConfiguration.toolchainDirPath + from: filePath.appending("usr"), + to: generator.pathsConfiguration.toolchainDirPath ) case .remoteTarball: try await generator.unpackHostSwift( @@ -287,7 +289,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) case let .localPackage(filePath): try await generator.copyTargetSwift( - from: filePath.appending("usr"), sdkDirPath: sdkDirPath + from: filePath.appending("usr"), + sdkDirPath: sdkDirPath ) case .remoteTarball: try await generator.unpackTargetSwiftPackage( @@ -314,7 +317,9 @@ public struct LinuxRecipe: SwiftSDKRecipe { // so they don't need this file. if self.versionsConfiguration.swiftVersion.hasAnyPrefix(from: ["5.9", "5.10", "6.0"]) { try await generator.generateSDKSettingsFile( - sdkDirPath: sdkDirPath, distribution: linuxDistribution) + sdkDirPath: sdkDirPath, + distribution: linuxDistribution + ) } if self.hostSwiftSource != .preinstalled { @@ -329,7 +334,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { } let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending( - "swift-autolink-extract") + "swift-autolink-extract" + ) if await !generator.doesFileExist(at: autolinkExtractPath) { logger.info("Fixing `swift-autolink-extract` symlink...") diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index c10f008..5161519 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -25,7 +25,8 @@ public struct SwiftSDKProduct { public protocol SwiftSDKRecipe: Sendable { /// Update the given toolset with platform specific options func applyPlatformOptions( - toolset: inout Toolset, targetTriple: Triple + toolset: inout Toolset, + targetTriple: Triple ) func applyPlatformOptions( metadata: inout SwiftSDKMetadataV4.TripleProperties, @@ -41,7 +42,9 @@ public protocol SwiftSDKRecipe: Sendable { /// The main entrypoint of the recipe to make a Swift SDK func makeSwiftSDK( - generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol + generator: SwiftSDKGenerator, + engine: QueryEngine, + httpClient: some HTTPClientProtocol ) async throws -> SwiftSDKProduct } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index dc3cb58..ca2b598 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -63,7 +63,8 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { toolset.swiftCompiler?.extraCLIOptions?.append( contentsOf: ccOptions.flatMap { ["-Xcc", $0] - }) + } + ) // Tell the C and C++ compilers to enable those features toolset.cCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) toolset.cxxCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) @@ -89,7 +90,8 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { var relativeToolchainDir = paths.toolchainDirPath guard relativeToolchainDir.removePrefix(paths.swiftSDKRootPath) else { fatalError( - "The toolchain bin directory path must be a subdirectory of the Swift SDK root path.") + "The toolchain bin directory path must be a subdirectory of the Swift SDK root path." + ) } metadata.swiftStaticResourcesPath = relativeToolchainDir.appending("usr/lib/swift_static").string @@ -109,7 +111,9 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { if let hostSwiftPackage { hostTriples = [hostSwiftPackage.triple] try await generator.rsync( - from: hostSwiftPackage.path.appending("usr"), to: pathsConfiguration.toolchainDirPath) + from: hostSwiftPackage.path.appending("usr"), + to: pathsConfiguration.toolchainDirPath + ) logger.info("Removing unused toolchain components...") let liblldbNames: [String] = try await { @@ -133,18 +137,23 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } else { // Simply copy the target Swift package into the SDK bundle when building host-agnostic SDK. try await generator.createDirectoryIfNeeded( - at: pathsConfiguration.toolchainDirPath.appending("usr")) + at: pathsConfiguration.toolchainDirPath.appending("usr") + ) try await generator.copy( - from: targetSwiftLibPath, to: pathsConfiguration.toolchainDirPath.appending("usr/lib")) + from: targetSwiftLibPath, + to: pathsConfiguration.toolchainDirPath.appending("usr/lib") + ) } let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending( - "swift-autolink-extract") + "swift-autolink-extract" + ) // WebAssembly object file requires `swift-autolink-extract` if await !generator.doesFileExist(at: autolinkExtractPath), await generator.doesFileExist( - at: generator.pathsConfiguration.toolchainBinDirPath.appending("swift")) + at: generator.pathsConfiguration.toolchainBinDirPath.appending("swift") + ) { logger.info("Fixing `swift-autolink-extract` symlink...") try await generator.createSymlink(at: autolinkExtractPath, pointingTo: "swift") @@ -191,7 +200,9 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { continue } try await generator.rsync( - from: distributionPath.appending(pathWithinPackage), to: pathWithinSwiftSDK) + from: distributionPath.appending(pathWithinPackage), + to: pathWithinSwiftSDK + ) } } } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index 7dca815..3f440e4 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -17,7 +17,8 @@ import NIOCore extension ByteBuffer { public func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { let result = try await ProcessExecutor.runCollectingOutput( - executable: zipPath, ["-cd"], + executable: zipPath, + ["-cd"], standardInput: [self].async, collectStandardOutput: true, collectStandardError: false, diff --git a/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift b/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift index b11d02e..a693d5d 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift @@ -26,7 +26,8 @@ public protocol HTTPClientProtocol: Sendable { /// NOTE: The client will be shutdown after the operation completes, so it /// should not be stored or used outside of the operation. static func with( - http1Only: Bool, _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + http1Only: Bool, + _ body: @Sendable (any HTTPClientProtocol) async throws -> Result ) async throws -> Result /// Download a file from the given URL to the given path. @@ -71,10 +72,12 @@ extension FilePath: @unchecked Sendable {} extension HTTPClient: HTTPClientProtocol { public static func with( - http1Only: Bool, _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + http1Only: Bool, + _ body: @Sendable (any HTTPClientProtocol) async throws -> Result ) async throws -> Result { var configuration = HTTPClient.Configuration( - redirectConfiguration: .follow(max: 5, allowCycles: false)) + redirectConfiguration: .follow(max: 5, allowCycles: false) + ) if http1Only { configuration.httpVersion = .http1Only } @@ -111,7 +114,8 @@ extension FilePath: @unchecked Sendable {} reportHead: { task, responseHead in if responseHead.status != .ok { task.fail( - reason: GeneratorError.fileDownloadFailed(url, responseHead.status.description)) + reason: GeneratorError.fileDownloadFailed(url, responseHead.status.description) + ) } } ) @@ -160,7 +164,9 @@ extension FilePath: @unchecked Sendable {} case let .success(finalProgress): continuation.yield( DownloadProgress( - totalBytes: finalProgress.totalBytes, receivedBytes: finalProgress.receivedBytes) + totalBytes: finalProgress.totalBytes, + receivedBytes: finalProgress.receivedBytes + ) ) continuation.finish() } @@ -175,7 +181,8 @@ extension FilePath: @unchecked Sendable {} struct OfflineHTTPClient: HTTPClientProtocol { static func with( - http1Only: Bool, _ body: @Sendable (any HTTPClientProtocol) async throws -> Result + http1Only: Bool, + _ body: @Sendable (any HTTPClientProtocol) async throws -> Result ) async throws -> Result { let client = OfflineHTTPClient() return try await body(client) @@ -200,11 +207,15 @@ struct OfflineHTTPClient: HTTPClientProtocol { status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer? ) { throw FileOperationError.downloadFailed( - URL(string: url)!, "Cannot fetch file with offline client") + URL(string: url)!, + "Cannot fetch file with offline client" + ) } public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { throw FileOperationError.downloadFailed( - URL(string: url)!, "Cannot fetch file with offline client") + URL(string: url)!, + "Cannot fetch file with offline client" + ) } } diff --git a/Sources/SwiftSDKGenerator/SystemUtils/which.swift b/Sources/SwiftSDKGenerator/SystemUtils/which.swift index 040a746..c856b46 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/which.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/which.swift @@ -20,7 +20,9 @@ import Foundation /// - Returns: The path to the executable if found, otherwise nil. func which(_ executableName: String) async throws -> String? { let result = try await ProcessExecutor.runCollectingOutput( - executable: "/usr/bin/which", [executableName], collectStandardOutput: true, + executable: "/usr/bin/which", + [executableName], + collectStandardOutput: true, collectStandardError: false, environment: ProcessInfo.processInfo.environment ) diff --git a/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift b/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift index 4f56cf0..d2d693e 100644 --- a/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift +++ b/Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift @@ -44,7 +44,8 @@ final class AsyncByteBufferLineSequenceTests: XCTestCase { for n in 0..<100 { let inputs: [ByteBuffer] = [ByteBuffer(repeating: 0, count: n)] let lines = try await Array( - inputs.async.splitIntoLines(dropLastChunkIfNoNewline: true).strings) + inputs.async.splitIntoLines(dropLastChunkIfNoNewline: true).strings + ) XCTAssertEqual([], lines) } } diff --git a/Tests/AsyncProcessTests/IntegrationTests.swift b/Tests/AsyncProcessTests/IntegrationTests.swift index 50d3c8a..922e37a 100644 --- a/Tests/AsyncProcessTests/IntegrationTests.swift +++ b/Tests/AsyncProcessTests/IntegrationTests.swift @@ -31,7 +31,8 @@ final class IntegrationTests: XCTestCase { func testTheBasicsWork() async throws { let exe = ProcessExecutor( group: self.group, - executable: "/bin/sh", ["-c", "exit 0"], + executable: "/bin/sh", + ["-c", "exit 0"], standardInput: EOFSequence(), logger: self.logger ) @@ -50,7 +51,8 @@ final class IntegrationTests: XCTestCase { for exitCode in UInt8.min...UInt8.max { let exe = ProcessExecutor( group: self.group, - executable: "/bin/sh", ["-c", "exit \(exitCode)"], + executable: "/bin/sh", + ["-c", "exit \(exitCode)"], standardInput: EOFSequence(), logger: self.logger ) @@ -77,7 +79,8 @@ final class IntegrationTests: XCTestCase { for signal in signalsToTest { let exe = ProcessExecutor( group: self.group, - executable: "/bin/sh", ["-c", "kill -\(signal) $$"], + executable: "/bin/sh", + ["-c", "kill -\(signal) $$"], standardInput: EOFSequence(), logger: self.logger ) @@ -99,7 +102,8 @@ final class IntegrationTests: XCTestCase { let input = AsyncStream.justMakeIt(elementType: ByteBuffer.self) let exe = ProcessExecutor( group: self.group, - executable: "/bin/cat", ["-nu"], // sh", ["-c", "while read -r line; do echo $line; done"], + executable: "/bin/cat", + ["-nu"], // sh", ["-c", "while read -r line; do echo $line; done"], standardInput: input.consumer, logger: self.logger ) @@ -209,7 +213,8 @@ final class IntegrationTests: XCTestCase { let input = AsyncStream.justMakeIt(elementType: ByteBuffer.self) let exe = ProcessExecutor( group: self.group, - executable: "/bin/sh", [], + executable: "/bin/sh", + [], standardInput: input.consumer, logger: self.logger ) @@ -531,7 +536,9 @@ final class IntegrationTests: XCTestCase { standardInput: EOFSequence(), logger: recordedLogger, logConfiguration: OutputLoggingSettings( - logLevel: .critical, to: .metadata(logMessage: "msg", key: "key")) + logLevel: .critical, + to: .metadata(logMessage: "msg", key: "key") + ) ).throwIfNonZero() XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.level == .critical }) XCTAssert(sharedRecorder.recordedMessages.allSatisfy { $0.message == "msg" }) @@ -611,7 +618,8 @@ final class IntegrationTests: XCTestCase { func testBasicRunMethodWorks() async throws { try await ProcessExecutor.run( group: self.group, - executable: "/bin/dd", ["if=/dev/zero", "bs=\(1024 * 1024)", "count=100"], + executable: "/bin/dd", + ["if=/dev/zero", "bs=\(1024 * 1024)", "count=100"], standardInput: EOFSequence(), logger: self.logger ).throwIfNonZero() @@ -620,7 +628,8 @@ final class IntegrationTests: XCTestCase { func testCollectJustStandardOutput() async throws { let allInfo = try await ProcessExecutor.runCollectingOutput( group: self.group, - executable: "/bin/dd", ["if=/dev/zero", "bs=\(1024 * 1024)", "count=1"], + executable: "/bin/dd", + ["if=/dev/zero", "bs=\(1024 * 1024)", "count=1"], standardInput: EOFSequence(), collectStandardOutput: true, collectStandardError: false, @@ -690,7 +699,8 @@ final class IntegrationTests: XCTestCase { do { let result = try await ProcessExecutor.runCollectingOutput( group: self.group, - executable: "/bin/dd", ["if=/dev/zero", "bs=\(1024 * 1024)", "count=1"], + executable: "/bin/dd", + ["if=/dev/zero", "bs=\(1024 * 1024)", "count=1"], standardInput: EOFSequence(), collectStandardOutput: true, collectStandardError: false, @@ -769,7 +779,8 @@ final class IntegrationTests: XCTestCase { func testAPIsWithoutELGOrLoggerArguments() async throws { let exe = ProcessExecutor( - executable: "/bin/sh", ["-c", "true"], + executable: "/bin/sh", + ["-c", "true"], standardInput: EOFSequence(), standardOutput: .discard, standardError: .discard @@ -777,7 +788,8 @@ final class IntegrationTests: XCTestCase { try await exe.run().throwIfNonZero() try await ProcessExecutor.run( - executable: "/bin/sh", ["-c", "true"], + executable: "/bin/sh", + ["-c", "true"], standardInput: EOFSequence() ).throwIfNonZero() @@ -807,7 +819,8 @@ final class IntegrationTests: XCTestCase { func testAPIsWithoutELGStandardInputOrLoggerArguments() async throws { let exe = ProcessExecutor( - executable: "/bin/sh", ["-c", "true"], + executable: "/bin/sh", + ["-c", "true"], standardOutput: .discard, standardError: .discard ) @@ -965,7 +978,8 @@ final class IntegrationTests: XCTestCase { ) { group in group.addTask { [logger = self.logger!] in try await ProcessExecutor.run( - executable: "/bin/sleep", ["100000"], + executable: "/bin/sleep", + ["100000"], logger: logger ) } diff --git a/Tests/GeneratorEngineTests/EngineTests.swift b/Tests/GeneratorEngineTests/EngineTests.swift index 5a3a95d..c3851e7 100644 --- a/Tests/GeneratorEngineTests/EngineTests.swift +++ b/Tests/GeneratorEngineTests/EngineTests.swift @@ -24,7 +24,9 @@ private let decoder = JSONDecoder() extension AsyncFileSystem { fileprivate func read( - _ path: FilePath, bufferLimit: Int = 10 * 1024 * 1024, as: V.Type + _ path: FilePath, + bufferLimit: Int = 10 * 1024 * 1024, + as: V.Type ) async throws -> V { let data = try await self.withOpenReadableFile(path) { var data = Data() diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index b067fd7..de885b6 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -19,7 +19,9 @@ import XCTest extension FileManager { func withTemporaryDirectory( - logger: Logger, cleanup: Bool = true, body: (URL) async throws -> T + logger: Logger, + cleanup: Bool = true, + body: (URL) async throws -> T ) async throws -> T { // Create a temporary directory using a UUID. Throws if the directory already exists. // The docs suggest using FileManager.url(for: .itemReplacementDirectory, ...) to create a temporary directory, @@ -74,7 +76,8 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: let installCommand = try XCTUnwrap( generatorOutput.split(separator: "\n").first { $0.contains("swift experimental-sdk install") - }) + } + ) let bundleName = try XCTUnwrap( FilePath(String(XCTUnwrap(installCommand.split(separator: " ").last))).components.last @@ -83,7 +86,8 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: logger.info("Checking installed Swift SDKs") let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components( - separatedBy: "\n") + separatedBy: "\n" + ) // Make sure this bundle hasn't been installed already. if installedSDKs.contains(bundleName) { @@ -136,10 +140,12 @@ final class RepeatedBuildTests: XCTestCase { do { try await Shell.run("docker ps") possibleArguments.append( - "--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9") + "--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9" + ) } catch { self.logger.warning( - "Docker CLI does not seem to be working, skipping tests that involve Docker.") + "Docker CLI does not seem to be working, skipping tests that involve Docker." + ) } for runArguments in possibleArguments { @@ -235,7 +241,9 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") let testPackageDir = FilePath(testPackageURL.path) try FileManager.default.createDirectory( - atPath: testPackageDir.string, withIntermediateDirectories: true) + atPath: testPackageDir.string, + withIntermediateDirectories: true + ) logger.info("Creating test project \(testPackageDir)") try await Shell.run("swift package --package-path \(testPackageDir) init --type executable") @@ -277,7 +285,8 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD logger.info("Test project built successfully") logger.info( - "Building test project in 6.0-\(containerVersion) container with static-swift-stdlib") + "Building test project in 6.0-\(containerVersion) container with static-swift-stdlib" + ) buildOutput = try await Shell.readStdout( """ docker run --rm -v \(testPackageDir):/src \ @@ -330,7 +339,10 @@ func buildTestcases(config: SDKConfiguration) async throws { let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in try await buildSDK( - logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) + logger, + scratchPath: tempDir.path, + withArguments: config.sdkGeneratorArguments + ) } logger.info("Built Swift SDK") @@ -345,7 +357,10 @@ func buildTestcases(config: SDKConfiguration) async throws { do { try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in try await buildTestcase( - logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir + logger, + testcase: testcase, + bundleName: bundleName, + tempDir: tempDir ) } } catch { @@ -469,13 +484,15 @@ final class Swift59_RHELEndToEndTests: XCTestCase { func testAmazonLinux2Aarch64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2")) + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) } func testAmazonLinux2X86_64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2")) + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) } } @@ -501,25 +518,29 @@ final class Swift510_RHELEndToEndTests: XCTestCase { func testAmazonLinux2Aarch64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2")) + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) } func testAmazonLinux2X86_64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2")) + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) } func testFedora39Aarch64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39")) + config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39") + ) } func testFedora39X86_64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39")) + config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39") + ) } } @@ -545,24 +566,28 @@ final class Swift60_RHELEndToEndTests: XCTestCase { func testAmazonLinux2Aarch64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2")) + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) } func testAmazonLinux2X86_64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2")) + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) } func testFedora39Aarch64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39")) + config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39") + ) } func testFedora39X86_64FromContainer() async throws { try skipSlow() try await buildTestcases( - config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39")) + config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39") + ) } } diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 656600e..01bef70 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -32,8 +32,10 @@ final class LinuxRecipeTests: XCTestCase { hostTriple: hostTriple, linuxDistribution: .init(name: .ubuntu, version: "22.04"), swiftVersion: swiftVersion, - swiftBranch: nil, lldVersion: "", - withDocker: withDocker, fromContainerImage: fromContainerImage, + swiftBranch: nil, + lldVersion: "", + withDocker: withDocker, + fromContainerImage: fromContainerImage, hostSwiftPackagePath: hostSwiftPackagePath, targetSwiftPackagePath: targetSwiftPackagePath, includeHostToolchain: includeHostToolchain, @@ -77,7 +79,8 @@ final class LinuxRecipeTests: XCTestCase { let recipe = try self.createRecipe(swiftVersion: testCase.swiftVersion) var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: testCase.targetTriple + toolset: &toolset, + targetTriple: testCase.targetTriple ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, testCase.expectedSwiftCompilerOptions) XCTAssertEqual(toolset.linker?.path, testCase.expectedLinkerPath) @@ -90,7 +93,8 @@ final class LinuxRecipeTests: XCTestCase { let recipe = try self.createRecipe(includeHostToolchain: false) var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: Triple("x86_64-unknown-linux-gnu") + toolset: &toolset, + targetTriple: Triple("x86_64-unknown-linux-gnu") ) XCTAssertEqual(toolset.rootPath, nil) XCTAssertEqual( @@ -98,14 +102,18 @@ final class LinuxRecipeTests: XCTestCase { [ "-Xlinker", "-R/usr/lib/swift/linux/", "-use-ld=lld", - ]) + ] + ) XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ["-lstdc++"]) XCTAssertEqual(toolset.librarian?.path, "llvm-ar") XCTAssert(toolset.linker == nil) } func runItemsToDownloadTestCase( - recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, includesHostSwift: Bool + recipe: LinuxRecipe, + includesHostLLVM: Bool, + includesTargetSwift: Bool, + includesHostSwift: Bool ) throws { let pathsConfiguration = PathsConfiguration( sourceRoot: ".", @@ -159,7 +167,10 @@ final class LinuxRecipeTests: XCTestCase { ( // Remote target tarball with preinstalled toolchain recipe: try createRecipe( - hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), + hostTriple: hostTriple, + swiftVersion: "5.9", + includeHostToolchain: false + ), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: false @@ -218,7 +229,10 @@ final class LinuxRecipeTests: XCTestCase { ( // Remote target tarball with preinstalled toolchain recipe: try createRecipe( - hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), + hostTriple: hostTriple, + swiftVersion: "5.9", + includeHostToolchain: false + ), includesTargetSwift: true, includesHostSwift: false ), diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift index 080a13b..228f4f5 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -32,7 +32,8 @@ final class WebAssemblyRecipeTests: XCTestCase { let recipe = self.createRecipe() var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: Triple("wasm32-unknown-wasi") + toolset: &toolset, + targetTriple: Triple("wasm32-unknown-wasi") ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, ["-static-stdlib"]) XCTAssertNil(toolset.cCompiler) @@ -44,7 +45,8 @@ final class WebAssemblyRecipeTests: XCTestCase { let recipe = self.createRecipe() var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( - toolset: &toolset, targetTriple: Triple("wasm32-unknown-wasip1-threads") + toolset: &toolset, + targetTriple: Triple("wasm32-unknown-wasip1-threads") ) XCTAssertEqual( toolset.swiftCompiler?.extraCLIOptions, @@ -69,6 +71,7 @@ final class WebAssemblyRecipeTests: XCTestCase { toolset.linker?.extraCLIOptions, [ "--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", - ]) + ] + ) } } From a3e3c73929d2334c4d7fdfeb30e3cc27cdfcc7c6 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Fri, 4 Apr 2025 05:40:34 -0400 Subject: [PATCH 08/17] Update defaults for Swift `6.1-RELEASE` tag (#204) - Defaulted `swiftVersion` to "6.1-RELEASE". - Added EndToEndTests for Ubuntu and RHEL for Swift61, including some additional tests for Ubuntu Jammy. - Fixed missing /lib and /lib64 directories in Ubuntu Noble when building from deb packages. If these directories don't exist, they are symlinked to the correct directories in /usr, which is consistent to what happens when copying from the Docker containers. - Added the fix for downloading Swift for the target that adds the platform name to the cache file, to avoid corruption of the artifacts: * image ALSO: I did not include EndToEndTests for Fedora39 for Swift 6.1 since the container is non-existent, likely because it's EOL. --- Sources/GeneratorCLI/GeneratorCLI.swift | 2 +- .../Artifacts/DownloadableArtifacts.swift | 10 +- .../SwiftSDKGenerator+Download.swift | 11 +++ .../VersionsConfiguration.swift | 23 ++--- .../SwiftSDKRecipes/LinuxRecipe.swift | 4 +- .../EndToEndTests.swift | 98 +++++++++++++++++++ 6 files changed, 130 insertions(+), 18 deletions(-) diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index cef47dc..ed4221a 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -114,7 +114,7 @@ extension GeneratorCLI { var swiftBranch: String? = nil @Option(help: "Version of Swift to supply in the bundle.") - var swiftVersion = "6.0.3-RELEASE" + var swiftVersion = "6.1-RELEASE" @Option( help: """ diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index ec4725e..1af7fd1 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -54,12 +54,12 @@ struct DownloadableArtifacts: Sendable { if hostTriple.os == .linux { // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts - let linuxArchSuffix = + let hostArchSuffix = hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" self.hostSwift = .init( remoteURL: versions.swiftDownloadURL( - subdirectory: "amazonlinux2\(linuxArchSuffix)", - platform: "amazonlinux2\(linuxArchSuffix)", + subdirectory: "amazonlinux2\(hostArchSuffix)", + platform: "amazonlinux2\(hostArchSuffix)", fileExtension: "tar.gz" ), localPath: paths.artifactsCachePath @@ -97,7 +97,9 @@ struct DownloadableArtifacts: Sendable { self.targetSwift = .init( remoteURL: versions.swiftDownloadURL(), localPath: paths.artifactsCachePath - .appending("target_swift_\(versions.swiftVersion)_\(targetTriple.triple).tar.gz"), + .appending( + "target_swift_\(versions.swiftVersion)_\(versions.swiftPlatform)_\(targetTriple.archName).tar.gz" + ), isPrebuilt: true ) } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 0807fe0..c3765e3 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -153,6 +153,17 @@ extension SwiftSDKGenerator { try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath) } } + + // Make sure we have /lib and /lib64, and if not symlink from /usr + // This makes building from packages more consistent with copying from the Docker container + let libDirectories = ["lib", "lib64"] + for dir in libDirectories { + let sdkLibPath = sdkDirPath.appending(dir) + let sdkUsrLibPath = sdkDirPath.appending("usr/\(dir)") + if !doesFileExist(at: sdkLibPath) && doesFileExist(at: sdkUsrLibPath) { + try createSymlink(at: sdkLibPath, pointingTo: FilePath("./usr/\(dir)")) + } + } } func downloadFiles( diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 88467aa..4bcaeca 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -37,14 +37,19 @@ public struct VersionsConfiguration: Sendable { var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): - return "ubuntu\(ubuntu.version)\(self.linuxArchSuffix)" + return "ubuntu\(ubuntu.version)" case let .rhel(rhel): - return "\(rhel.rawValue)\(self.linuxArchSuffix)" + return rhel.rawValue } } + var swiftPlatformAndSuffix: String { + return "\(self.swiftPlatform)\(self.linuxArchSuffix)" + } + func swiftDistributionName(platform: String? = nil) -> String { - "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatform)" + return + "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatformAndSuffix)" } func swiftDownloadURL( @@ -52,14 +57,10 @@ public struct VersionsConfiguration: Sendable { platform: String? = nil, fileExtension: String = "tar.gz" ) -> URL { - let computedSubdirectory: String - switch self.linuxDistribution { - case let .ubuntu(ubuntu): - computedSubdirectory = - "ubuntu\(ubuntu.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" - case let .rhel(rhel): - computedSubdirectory = rhel.rawValue - } + let computedPlatform = platform ?? self.swiftPlatformAndSuffix + let computedSubdirectory = + subdirectory + ?? computedPlatform.replacingOccurrences(of: ".", with: "") return URL( string: """ diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 923462d..ed13996 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -178,7 +178,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { var items: [DownloadableArtifacts.Item] = [] if self.hostSwiftSource != .preinstalled && self.mainHostTriple.os != .linux - && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.") { items.append(artifacts.hostLLVM) } @@ -324,7 +324,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { if self.hostSwiftSource != .preinstalled { if self.mainHostTriple.os != .linux - && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.") { try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM) } diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index de885b6..443fa5b 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -191,6 +191,12 @@ struct SDKConfiguration { return res } + func withLinuxDistributionVersion(_ version: String) -> SDKConfiguration { + var res = self + res.linuxDistributionVersion = version + return res + } + func withArchitecture(_ arch: String) -> SDKConfiguration { var res = self res.architecture = arch @@ -462,6 +468,64 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { } } +final class Swift61_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "ubuntu", + linuxDistributionVersion: "24.04", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testJammyAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withLinuxDistributionVersion("22.04") + ) + } + + func testJammyX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withLinuxDistributionVersion("22.04") + ) + } + + func testJammyAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withLinuxDistributionVersion("22.04").withDocker() + ) + } + + func testJammyX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withLinuxDistributionVersion("22.04").withDocker() + ) + } +} + final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", @@ -591,3 +655,37 @@ final class Swift60_RHELEndToEndTests: XCTestCase { ) } } + +final class Swift61_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) + } +} From 9fe010c8cc8349675a3f2abc48f7838571a570c8 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Wed, 16 Apr 2025 11:05:08 +0100 Subject: [PATCH 09/17] docs: Delete instructions to run non-existent soundness.sh (#206) Utilities/soundness.sh was deleted in #199 which moved CI to GitHub Actions. Credit to @heckj, who found the same problem in https://github.com/apple/swift-container-plugin/issues/96. --- CONTRIBUTING.md | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2acd53b..10ce9c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,29 +58,6 @@ We require that your commit messages match our template. The easiest way to do t git config commit.template Utilities/git.commit.template -### Run `./Utilities/soundness.sh` - -The scripts directory contains a [`soundness.sh` script](https://github.com/apple/swift-sdk-generator/blob/main/Utilities/soundness.sh) -that enforces additional checks, like license headers and formatting style. - -Please make sure to run `./Utilities/soundness.sh` before pushing a change upstream, otherwise it is likely the PR validation will fail -on minor changes such as formatting issues. - -For frequent contributors, we recommend adding the script as a [git pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks), which you can do via executing the following command in the project root directory: - -```bash -cat << EOF > .git/hooks/pre-push - -if [[ -f "Utilities/soundness.sh" ]]; then - Utilities/soundness.sh -fi -EOF -``` - -Which makes the script execute, and only allow the `git push` to complete if the check has passed. - -In the case of formatting issues, you can then `git add` the formatting changes, and attempt the push again. - ## How to contribute your work -Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. \ No newline at end of file +Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. From ed6a85b1fed5036b2cfc0f242c66e08c5c0d45ed Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 16 Apr 2025 06:23:10 -0400 Subject: [PATCH 10/17] Support Debian 11 and Debian 12 distributions in the generator (#203) This completes #116 and can be used to generate Debian Swift SDKs either by downloading dependencies from mirrors directly or from containers. - Added Debian `bullseye` and `bookworm` to `LinuxDistribution` and modded `SwiftSDKGenerator+Download` to work downloading both Ubuntu and Debian packages from their respective mirrors. - Swift SDKs can be generated from Docker using the `swift:*-bookworm` container for Debian 12, but for Debian 11 you must create the container yourself/manually put the Swift toolchain into it. - Added special handling in `VersionsConfiguration` so that if Debian 11 (bullseye) is selected, the Ubuntu 20.04 toolchain is downloaded. If Debian 12 (bookworm) is selected for Swift 5.9.* or Swift 5.10, the Ubuntu 22.04 toolchain is used instead. https://askubuntu.com/a/445496 - Added tests for the special handling in `LinuxRecipeTests.testItemsToDownloadForDebianTargets()` to ensure the correct toolchain is selected. To use: ```bash $ swift run swift-sdk-generator make-linux-sdk --linux-distribution-name debian --linux-distribution-version 11 $ swift run swift-sdk-generator make-linux-sdk --linux-distribution-name debian --linux-distribution-version 12 $ swift run swift-sdk-generator make-linux-sdk --linux-distribution-name debian --linux-distribution-version 12 --with-docker ``` I also added EndToEndTests for all Debian supported combinations, resulting in another 30GB of generated Swift SDKs: ``` 2.1 GiB [####################] /debian_12_x86_64_6.0.3-RELEASE_with-docker.artifactbundle 2.1 GiB [################### ] /debian_12_aarch64_6.0.3-RELEASE_with-docker.artifactbundle 2.0 GiB [################## ] /debian_12_x86_64_5.10.1-RELEASE_with-docker.artifactbundle 1.9 GiB [################# ] /debian_12_x86_64_6.0.3-RELEASE.artifactbundle 1.9 GiB [################# ] /debian_12_aarch64_6.0.3-RELEASE.artifactbundle 1.9 GiB [################# ] /debian_11_x86_64_6.0.3-RELEASE.artifactbundle 1.9 GiB [################# ] /debian_12_aarch64_5.10.1-RELEASE_with-docker.artifactbundle 1.9 GiB [################# ] /debian_11_aarch64_6.0.3-RELEASE.artifactbundle 1.8 GiB [################ ] /debian_12_x86_64_5.10.1-RELEASE.artifactbundle 1.7 GiB [################ ] /debian_12_aarch64_5.10.1-RELEASE.artifactbundle 1.7 GiB [################ ] /debian_11_x86_64_5.10.1-RELEASE.artifactbundle 1.7 GiB [############### ] /debian_11_aarch64_5.10.1-RELEASE.artifactbundle 1.7 GiB [############### ] /debian_12_x86_64_5.9.2-RELEASE.artifactbundle 1.7 GiB [############### ] /debian_12_aarch64_5.9.2-RELEASE.artifactbundle 1.7 GiB [############### ] /debian_11_x86_64_5.9.2-RELEASE.artifactbundle 1.6 GiB [############### ] /debian_11_aarch64_5.9.2-RELEASE.artifactbundle ``` To make this work properly, I modded the `targetSwift` path in `DownloadableArtifacts` to use the swiftPlatform name in the file name to avoid the issues I saw with the EndToEndTests, where the *.tar.gz files would get corrupted when trying to download a different version of the target Swift on top of the existing file. Now, they look like this: ``` target_swift_5.10.1-RELEASE_debian12_aarch64.tar.gz target_swift_5.9.2-RELEASE_ubuntu22.04_aarch64.tar.gz target_swift_5.10.1-RELEASE_debian12_x86_64.tar.gz target_swift_5.9.2-RELEASE_ubuntu22.04_x86_64.tar.gz target_swift_5.10.1-RELEASE_ubuntu20.04_aarch64.tar.gz target_swift_6.0.3-RELEASE_debian12_aarch64.tar.gz target_swift_5.10.1-RELEASE_ubuntu20.04_x86_64.tar.gz target_swift_6.0.3-RELEASE_debian12_x86_64.tar.gz target_swift_5.10-RELEASE_ubuntu20.04_aarch64.tar.gz target_swift_6.0.3-RELEASE_ubuntu20.04_aarch64.tar.gz target_swift_5.9.2-RELEASE_ubuntu20.04_aarch64.tar.gz target_swift_6.0.3-RELEASE_ubuntu20.04_x86_64.tar.gz target_swift_5.9.2-RELEASE_ubuntu20.04_x86_64.tar.gz ``` --- README.md | 6 +- Sources/GeneratorCLI/GeneratorCLI.swift | 10 +- .../Generator/SwiftSDKGenerator+Copy.swift | 37 ++- .../SwiftSDKGenerator+Download.swift | 265 ++++++++++-------- .../PlatformModels/LinuxDistribution.swift | 70 ++++- .../VersionsConfiguration.swift | 21 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 30 +- .../SystemUtils/ByteBuffer+Utils.swift | 6 +- .../SystemUtils/GeneratorError.swift | 16 +- .../EndToEndTests.swift | 178 ++++++++++++ .../SwiftSDKRecipes/LinuxRecipeTests.swift | 132 ++++++++- 11 files changed, 615 insertions(+), 156 deletions(-) diff --git a/README.md b/README.md index 5ec9617..4894c5c 100644 --- a/README.md +++ b/README.md @@ -47,14 +47,14 @@ The generator also allows cross-compiling between any Linux distributions offici | macOS (arm64) | ✅ macOS 13.0+ | ❌ | | macOS (x86_64) | ✅ macOS 13.0+[^1] | ❌ | | Ubuntu | ✅ 20.04+ | ✅ 20.04+ | -| RHEL | ✅ Fedora 39[^2], UBI 9 | ✅ UBI 9 | +| Debian | ✅ 11, 12[^2] | ✅ 11, 12[^2] | +| RHEL | ✅ Fedora 39, UBI 9 | ✅ Fedora 39, UBI 9[^3] | | Amazon Linux 2 | ✅ Supported | ✅ Supported[^3] | -| Debian 12 | ✅ Supported[^2] | ✅ Supported[^2][^3] | [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). -[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms. +[^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible. [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138). ## How to use it diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index ed4221a..0724b1b 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -199,7 +199,8 @@ extension GeneratorCLI { @Option( help: """ - Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`. + Linux distribution to use if the target platform is Linux. + - Available options: `ubuntu`, `debian`, `rhel`. Default is `ubuntu`. """, transform: LinuxDistribution.Name.init(nameString:) ) @@ -208,8 +209,9 @@ extension GeneratorCLI { @Option( help: """ Version of the Linux distribution used as a target platform. - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). + - Available options for Ubuntu: `20.04`, `22.04` (default when `--distribution-name` is `ubuntu`), `24.04`. + - Available options for Debian: `11`, `12` (default when `--distribution-name` is `debian`). + - Available options for RHEL: `ubi9` (default when `--distribution-name` is `rhel`). """ ) var linuxDistributionVersion: String? @@ -239,6 +241,8 @@ extension GeneratorCLI { linuxDistributionDefaultVersion = "ubi9" case .ubuntu: linuxDistributionDefaultVersion = "22.04" + case .debian: + linuxDistributionDefaultVersion = "12" } let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 3942662..b6d9d7c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -22,6 +22,7 @@ extension SwiftSDKGenerator { logger.info("Launching a container to extract the Swift SDK for the target triple...") try await withDockerContainer(fromImage: baseDockerImage) { containerID in try await inTemporaryDirectory { generator, _ in + let sdkLibPath = sdkDirPath.appending("lib") let sdkUsrPath = sdkDirPath.appending("usr") try await generator.createDirectoryIfNeeded(at: sdkUsrPath) try await generator.copyFromDockerContainer( @@ -60,6 +61,15 @@ extension SwiftSDKGenerator { to: sdkUsrLib64Path ) try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") + } else if case let containerLib64 = FilePath("/lib64"), + try await generator.doesPathExist(containerLib64, inContainer: containerID) + { + let sdkLib64Path = sdkDirPath.appending("lib64") + try await generator.copyFromDockerContainer( + id: containerID, + from: containerLib64, + to: sdkLib64Path + ) } let sdkUsrLibPath = sdkUsrPath.appending("lib") @@ -72,13 +82,26 @@ extension SwiftSDKGenerator { // architecture-specific directories: // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. - if case .ubuntu = targetDistribution { - subpaths += [("\(targetTriple.archName)-linux-gnu", false)] + if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { + var archSubpath = "\(targetTriple.archName)-linux-gnu" - // Custom subpath for armv7 + // armv7 with Debian uses a custom subpath for armhf if targetTriple.archName == "armv7" { - subpaths += [("arm-linux-gnueabihf", false)] + archSubpath = "arm-linux-gnueabihf" } + + // Copy /lib/ for Debian 11 + if case let .debian(debian) = targetDistribution, debian == .bullseye { + try await generator.createDirectoryIfNeeded(at: sdkLibPath) + try await generator.copyFromDockerContainer( + id: containerID, + from: FilePath("/lib").appending(archSubpath), + to: sdkLibPath.appending(archSubpath), + failIfNotExists: false + ) + } + + subpaths += [(archSubpath, false)] } for (subpath, failIfNotExists) in subpaths { @@ -89,7 +112,11 @@ extension SwiftSDKGenerator { failIfNotExists: failIfNotExists ) } - try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib") + + // Symlink if we do not have a /lib directory in the SDK + if await !generator.doesFileExist(at: sdkLibPath) { + try await generator.createSymlink(at: sdkLibPath, pointingTo: "usr/lib") + } // Look for 32-bit libraries to remove from RHEL-based distros // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index c3765e3..d7f1935 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -20,8 +20,9 @@ import class Foundation.FileManager import struct Foundation.URL import struct SystemPackage.FilePath -private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" -private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" +private let ubuntuMainMirror = "http://gb.archive.ubuntu.com/ubuntu" +private let ubuntuPortsMirror = "http://ports.ubuntu.com/ubuntu-ports" +private let debianMirror = "http://deb.debian.org/debian" extension FilePath { var metadataValue: Logger.MetadataValue { @@ -80,18 +81,47 @@ extension SwiftSDKGenerator { ) } - func downloadUbuntuPackages( + func getMirrorURL(for linuxDistribution: LinuxDistribution) throws -> String { + if linuxDistribution.name == .ubuntu { + if targetTriple.arch == .x86_64 { + return ubuntuMainMirror + } else { + return ubuntuPortsMirror + } + } else if linuxDistribution.name == .debian { + return debianMirror + } else { + throw GeneratorError.distributionSupportsOnlyDockerGenerator(linuxDistribution) + } + } + + func packagesFileName(isXzAvailable: Bool) -> String { + if isXzAvailable { + return "Packages.xz" + } + // Use .gz if xz is not available + return "Packages.gz" + } + + func downloadDebianPackages( _ client: some HTTPClientProtocol, _ engine: QueryEngine, requiredPackages: [String], versionsConfiguration: VersionsConfiguration, sdkDirPath: FilePath ) async throws { - logger.debug("Parsing Ubuntu packages list...") + let mirrorURL = try getMirrorURL(for: versionsConfiguration.linuxDistribution) + let distributionName = versionsConfiguration.linuxDistribution.name + let distributionRelease = versionsConfiguration.linuxDistribution.release // Find xz path let xzPath = try await which("xz") if xzPath == nil { + // If we don't have xz, it's required for Packages.xz for debian + if distributionName == .debian { + throw GeneratorError.debianPackagesListDownloadRequiresXz + } + logger.warning( """ The `xz` utility was not found in `PATH`. \ @@ -100,49 +130,73 @@ extension SwiftSDKGenerator { ) } - async let mainPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) - - async let updatesPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath + logger.info( + "Downloading and parsing packages lists...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "distributionRelease": .string(distributionRelease), + ] ) - async let universePackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "universe", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) + let allPackages = try await withThrowingTaskGroup(of: [String: URL].self) { group in + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + if distributionName == .ubuntu { + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "universe", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + } - let allPackages = - try await mainPackages - .merging(updatesPackages, uniquingKeysWith: { $1 }) - .merging(universePackages, uniquingKeysWith: { $1 }) + var packages: [String: URL] = [String: URL]() + for try await result in group { + packages.merge(result, uniquingKeysWith: { $1 }) + } + return packages + } let urls = requiredPackages.compactMap { allPackages[$0] } guard urls.count == requiredPackages.count else { - throw GeneratorError.ubuntuPackagesParsingFailure( + throw GeneratorError.packagesListParsingFailure( expectedPackages: requiredPackages.count, actual: urls.count ) } logger.info( - "Downloading Ubuntu packages...", - metadata: ["packageCount": .stringConvertible(urls.count)] + "Downloading packages...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "packageCount": .stringConvertible(urls.count), + ] ) try await inTemporaryDirectory { fs, tmpDir in let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine) @@ -166,6 +220,67 @@ extension SwiftSDKGenerator { } } + private func parseDebianPackageList( + using client: HTTPClientProtocol, + mirrorURL: String, + release: String, + releaseSuffix: String, + repository: String, + targetTriple: Triple, + xzPath: String? + ) async throws -> [String: URL] { + var contextLogger = logger + + let packagesListURL = """ + \(mirrorURL)/dists/\(release)\(releaseSuffix)/\(repository)/binary-\( + targetTriple.arch!.debianConventionName + )/\(packagesFileName(isXzAvailable: xzPath != nil)) + """ + contextLogger[metadataKey: "packagesListURL"] = .string(packagesListURL) + + contextLogger.debug("Downloading packages list...") + guard + let packages = try await client.downloadDebianPackagesList( + from: packagesListURL, + unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available + logger: logger + ) + else { + throw GeneratorError.packagesListDecompressionFailure + } + + let packageRef = Reference(Substring.self) + let pathRef = Reference(Substring.self) + + let regex = Regex { + "Package: " + + Capture(as: packageRef) { + OneOrMore(.anyNonNewline) + } + + OneOrMore(.any, .reluctant) + + "Filename: " + + Capture(as: pathRef) { + OneOrMore(.anyNonNewline) + } + + Anchor.endOfLine + } + + contextLogger.debug("Processing packages list...") + var result = [String: URL]() + for match in packages.matches(of: regex) { + guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } + + result[String(match[packageRef])] = url + } + + return result + } + func downloadFiles( from urls: [URL], to directory: FilePath, @@ -218,92 +333,16 @@ extension SwiftSDKGenerator { } extension HTTPClientProtocol { - private func downloadUbuntuPackagesList( + func downloadDebianPackagesList( from url: String, unzipWith zipPath: String, - isVerbose: Bool + logger: Logger ) async throws -> String? { - guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) + guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, logger: logger) else { throw FileOperationError.downloadFailed(url) } return String(buffer: packages) } - - func packagesFileName(isXzAvailable: Bool) -> String { - if isXzAvailable { - return "Packages.xz" - } - // Use .gz if xz is not available - return "Packages.gz" - } - - func parseUbuntuPackagesList( - ubuntuRelease: String, - releaseSuffix: String = "", - repository: String, - targetTriple: Triple, - isVerbose: Bool, - xzPath: String? - ) async throws -> [String: URL] { - let mirrorURL: String - if targetTriple.arch == .x86_64 { - mirrorURL = ubuntuAMD64Mirror - } else { - mirrorURL = ubuntuARM64Mirror - } - - let packagesListURL = """ - \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\( - targetTriple.arch!.debianConventionName - )/\(packagesFileName(isXzAvailable: xzPath != nil)) - """ - - guard - let packages = try await downloadUbuntuPackagesList( - from: packagesListURL, - unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available - isVerbose: isVerbose - ) - else { - throw GeneratorError.ubuntuPackagesDecompressionFailure - } - - let packageRef = Reference(Substring.self) - let pathRef = Reference(Substring.self) - - let regex = Regex { - "Package: " - - Capture(as: packageRef) { - OneOrMore(.anyNonNewline) - } - - OneOrMore(.any, .reluctant) - - "Filename: " - - Capture(as: pathRef) { - OneOrMore(.anyNonNewline) - } - - Anchor.endOfLine - - OneOrMore(.any, .reluctant) - - "Description-md5: " - - OneOrMore(.hexDigit) - } - - var result = [String: URL]() - for match in packages.matches(of: regex) { - guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } - - result[String(match[packageRef])] = url - } - - return result - } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index 5385057..7f64509 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -11,9 +11,10 @@ //===----------------------------------------------------------------------===// public enum LinuxDistribution: Hashable, Sendable { - public enum Name: String { + public enum Name: String, Sendable { case rhel case ubuntu + case debian } public enum RHEL: String, Sendable { @@ -97,8 +98,66 @@ public enum LinuxDistribution: Hashable, Sendable { } } + public enum Debian: String, Sendable { + case bullseye + case bookworm + + init(version: String) throws { + switch version { + case "11": self = .bullseye + case "12": self = .bookworm + default: + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.debian.rawValue, + version: version + ) + } + } + + var version: String { + switch self { + case .bullseye: return "11" + case .bookworm: return "12" + } + } + + public var requiredPackages: [String] { + switch self { + case .bullseye: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-10-dev", + "libicu67", + "libicu-dev", + "libstdc++-10-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + case .bookworm: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-12-dev", + "libicu72", + "libicu-dev", + "libstdc++-12-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + } + } + } + case rhel(RHEL) case ubuntu(Ubuntu) + case debian(Debian) public init(name: Name, version: String) throws { switch name { @@ -110,6 +169,9 @@ public enum LinuxDistribution: Hashable, Sendable { case .ubuntu: self = try .ubuntu(Ubuntu(version: version)) + + case .debian: + self = try .debian(Debian(version: version)) } } @@ -117,6 +179,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case .rhel: return .rhel case .ubuntu: return .ubuntu + case .debian: return .debian } } @@ -124,6 +187,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return rhel.rawValue case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } @@ -131,6 +195,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return "rhel-\(rhel.rawValue)" case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } } @@ -150,7 +215,7 @@ extension LinuxDistribution: CustomStringConvertible { switch self { case .rhel: versionComponent = self.release.uppercased() - case .ubuntu: + case .ubuntu, .debian: versionComponent = self.release.capitalized } @@ -163,6 +228,7 @@ extension LinuxDistribution.Name: CustomStringConvertible { switch self { case .rhel: return "RHEL" case .ubuntu: return "Ubuntu" + case .debian: return "Debian" } } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 4bcaeca..3459126 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import Logging + import struct Foundation.URL public struct VersionsConfiguration: Sendable { @@ -18,7 +20,8 @@ public struct VersionsConfiguration: Sendable { swiftBranch: String? = nil, lldVersion: String, linuxDistribution: LinuxDistribution, - targetTriple: Triple + targetTriple: Triple, + logger: Logger ) throws { self.swiftVersion = swiftVersion self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" @@ -26,6 +29,7 @@ public struct VersionsConfiguration: Sendable { self.linuxDistribution = linuxDistribution self.linuxArchSuffix = targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + self.logger = logger } let swiftVersion: String @@ -33,11 +37,21 @@ public struct VersionsConfiguration: Sendable { let lldVersion: String let linuxDistribution: LinuxDistribution let linuxArchSuffix: String + private let logger: Logger var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): return "ubuntu\(ubuntu.version)" + case let .debian(debian): + if debian.version == "11" { + // Ubuntu 20.04 toolchain is binary compatible with Debian 11 + return "ubuntu20.04" + } else if self.swiftVersion.hasPrefix("5.9") || self.swiftVersion == "5.10" { + // Ubuntu 22.04 toolchain is binary compatible with Debian 12 + return "ubuntu22.04" + } + return "debian\(debian.version)" case let .rhel(rhel): return rhel.rawValue } @@ -66,9 +80,8 @@ public struct VersionsConfiguration: Sendable { string: """ https://download.swift.org/\( self.swiftBranch - )/\( - subdirectory ?? computedSubdirectory - )/swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: platform)).\(fileExtension) + )/\(computedSubdirectory)/\ + swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: computedPlatform)).\(fileExtension) """ )! } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index ed13996..9c767d7 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -63,7 +63,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { swiftBranch: swiftBranch, lldVersion: lldVersion, linuxDistribution: linuxDistribution, - targetTriple: targetTriple + targetTriple: targetTriple, + logger: logger ) let targetSwiftSource: LinuxRecipe.TargetSwiftSource @@ -251,19 +252,28 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) if !self.shouldUseDocker { - guard case let .ubuntu(version) = linuxDistribution else { + switch linuxDistribution { + case .ubuntu(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + case .debian(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + default: throw GeneratorError .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) } - - try await generator.downloadUbuntuPackages( - client, - engine, - requiredPackages: version.requiredPackages, - versionsConfiguration: self.versionsConfiguration, - sdkDirPath: sdkDirPath - ) } switch self.hostSwiftSource { diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index 3f440e4..f8f1414 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -12,17 +12,19 @@ import AsyncProcess import Foundation +import Logging import NIOCore extension ByteBuffer { - public func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { + public func unzip(zipPath: String, logger: Logger) async throws -> ByteBuffer? { let result = try await ProcessExecutor.runCollectingOutput( executable: zipPath, ["-cd"], standardInput: [self].async, collectStandardOutput: true, collectStandardError: false, - perStreamCollectionLimitBytes: 20 * 1024 * 1024 + perStreamCollectionLimitBytes: 100 * 1024 * 1024, + logger: logger ) try result.exitReason.throwIfNonZero() diff --git a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift index e7706de..6fcdb7b 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift @@ -25,8 +25,9 @@ enum GeneratorError: Error { case distributionDoesNotSupportArchitecture(LinuxDistribution, targetArchName: String) case fileDoesNotExist(FilePath) case fileDownloadFailed(URL, String) - case ubuntuPackagesDecompressionFailure - case ubuntuPackagesParsingFailure(expectedPackages: Int, actual: Int) + case debianPackagesListDownloadRequiresXz + case packagesListDecompressionFailure + case packagesListParsingFailure(expectedPackages: Int, actual: Int) } extension GeneratorError: CustomStringConvertible { @@ -61,11 +62,12 @@ extension GeneratorError: CustomStringConvertible { case let .fileDownloadFailed(url, status): return "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." - case .ubuntuPackagesDecompressionFailure: - return "Failed to decompress the list of Ubuntu packages" - case let .ubuntuPackagesParsingFailure(expected, actual): - return - "Failed to parse Ubuntu packages manifest, expected \(expected), found \(actual) packages." + case .debianPackagesListDownloadRequiresXz: + return "Downloading the Debian packages list requires xz, and it is not installed." + case .packagesListDecompressionFailure: + return "Failed to decompress the list of packages." + case let .packagesListParsingFailure(expected, actual): + return "Failed to parse packages manifest, expected \(expected), found \(actual) packages." } } } diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 443fa5b..9a9222f 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -526,6 +526,184 @@ final class Swift61_UbuntuEndToEndTests: XCTestCase { } } +final class Swift59_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "debian", + linuxDistributionVersion: "11", // we use ubuntu2004 toolchain here + architecture: "aarch64", + withDocker: false + ) + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("aarch64") + ) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("x86_64") + ) + } + + // NOTE: the generator does not support building a Debian 11/Debian 12 Swift SDK with Docker + // for Swift 5.9.x and 5.10 without a pre-built container, so we do not test this here. +} + +final class Swift510_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift60_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.3", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift61_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 01bef70..ddda1de 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -20,6 +20,7 @@ final class LinuxRecipeTests: XCTestCase { func createRecipe( hostTriple: Triple = Triple("x86_64-unknown-linux-gnu"), + linuxDistribution: LinuxDistribution, swiftVersion: String = "6.0", withDocker: Bool = false, fromContainerImage: String? = nil, @@ -30,7 +31,7 @@ final class LinuxRecipeTests: XCTestCase { try LinuxRecipe( targetTriple: Triple("aarch64-unknown-linux-gnu"), hostTriple: hostTriple, - linuxDistribution: .init(name: .ubuntu, version: "22.04"), + linuxDistribution: linuxDistribution, swiftVersion: swiftVersion, swiftBranch: nil, lldVersion: "", @@ -75,8 +76,12 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { - let recipe = try self.createRecipe(swiftVersion: testCase.swiftVersion) + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + swiftVersion: testCase.swiftVersion + ) var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, @@ -90,7 +95,11 @@ final class LinuxRecipeTests: XCTestCase { } func testToolOptionsForPreinstalledSdk() throws { - let recipe = try self.createRecipe(includeHostToolchain: false) + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + includeHostToolchain: false + ) var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( toolset: &toolset, @@ -145,6 +154,7 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForMacOSHost() throws { let hostTriple = Triple("x86_64-apple-macos") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases: [( recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, @@ -152,14 +162,22 @@ final class LinuxRecipeTests: XCTestCase { )] = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesHostLLVM: true, includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: true @@ -168,6 +186,7 @@ final class LinuxRecipeTests: XCTestCase { // Remote target tarball with preinstalled toolchain recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.9", includeHostToolchain: false ), @@ -179,6 +198,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -191,6 +211,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -213,16 +234,25 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForLinuxHost() throws { let hostTriple = Triple("x86_64-unknown-linux-gnu") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesTargetSwift: true, includesHostSwift: true ), @@ -230,6 +260,7 @@ final class LinuxRecipeTests: XCTestCase { // Remote target tarball with preinstalled toolchain recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.9", includeHostToolchain: false ), @@ -240,6 +271,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -251,6 +283,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -270,6 +303,89 @@ final class LinuxRecipeTests: XCTestCase { } } + // Ubuntu toolchains will be selected for Debian 11 and 12 depending on the Swift version + func testItemsToDownloadForDebianTargets() throws { + let hostTriple = Triple("x86_64-unknown-linux-gnu") + let testCases = [ + ( + // Debian 11 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.9 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 12 with Swift 5.10 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 11 with Swift 6.0 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.10.1 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10.1" + ), + expectedTargetSwift: "debian12" + ), + ( + // Debian 12 with Swift 6.0 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "debian12" + ), + ] + + for testCase in testCases { + + let pathsConfiguration = PathsConfiguration( + sourceRoot: ".", + artifactID: "my-sdk", + targetTriple: testCase.recipe.mainTargetTriple + ) + let downloadableArtifacts = try DownloadableArtifacts( + hostTriple: testCase.recipe.mainHostTriple, + targetTriple: testCase.recipe.mainTargetTriple, + testCase.recipe.versionsConfiguration, + pathsConfiguration + ) + let itemsToDownload = testCase.recipe.itemsToDownload(from: downloadableArtifacts) + let targetSwiftRemoteURL = itemsToDownload.first(where: { + $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL + })?.remoteURL.absoluteString + + // If this is a Linux host, we do not download LLVM + XCTAssert(targetSwiftRemoteURL!.contains(testCase.expectedTargetSwift)) + } + } + func testHostTriples() throws { let allHostTriples = [ Triple("x86_64-unknown-linux-gnu"), @@ -287,8 +403,10 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { let recipe = try createRecipe( + linuxDistribution: linuxDistribution, swiftVersion: testCase.swiftVersion, includeHostToolchain: testCase.includeHostToolchain ) From 44ea35b383a9f362156b6c49cb6aa425574fb1aa Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 14 May 2025 17:38:45 +0100 Subject: [PATCH 11/17] Bump `// swift-tools-version` to 5.9 (#207) All of the CI jobs are using Swift 5.9 or later at this point. --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 832b742..479975e 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.8 +// swift-tools-version: 5.9 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription From 19deeb1f36715428cf36dc5c8514d5c124d99156 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 15 May 2025 12:30:52 +0100 Subject: [PATCH 12/17] Bump `lineLength` to 120 in `.swift-format` (#211) This seems to be a reasonable limit still readable on most devices, while reducing the amount of reformatting applied by `swift format` checks. --- .swift-format | 2 +- .../SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.swift-format b/.swift-format index 3b4cc63..e9b435d 100644 --- a/.swift-format +++ b/.swift-format @@ -11,7 +11,7 @@ "lineBreakBeforeControlFlowKeywords" : false, "lineBreakBeforeEachArgument" : true, "lineBreakBeforeEachGenericRequirement" : false, - "lineLength" : 100, + "lineLength" : 120, "maximumBlankLines" : 1, "multiElementCollectionTrailingCommas" : true, "noAssignmentInExpressions" : { diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index ca2b598..f3b16dc 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -167,8 +167,7 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } /// Merge the target Swift package into the Swift SDK bundle derived from the host Swift package. - func mergeTargetSwift(from distributionPath: FilePath, generator: SwiftSDKGenerator) async throws - { + func mergeTargetSwift(from distributionPath: FilePath, generator: SwiftSDKGenerator) async throws { let pathsConfiguration = generator.pathsConfiguration logger.info("Copying Swift core libraries for the target triple into Swift SDK bundle...") for (pathWithinPackage, pathWithinSwiftSDK, isOptional) in [ From fc7da523b2d06eea002b949c7a4464b190002532 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 15 May 2025 13:46:18 +0100 Subject: [PATCH 13/17] Generate Embedded Swift SDK for WASI (#208) This allows targeting WASI with Embedded Swift, enabling use of a subset of Swift Concurrency in this mode, and also hiding most if not all of unsafe flags in toolsets. This way, users of Embedded Swift can build their packages with a similar `swift build --swift-sdk` command as the rest of users of WASI. --- .../Helpers/Vendor/QueryEngine/CacheKey.swift | 1 - .../SwiftSDKGenerator+Entrypoint.swift | 30 ++++++--- .../SwiftSDKGenerator+Metadata.swift | 60 +++++++++++------- .../Serialization/SwiftSDKMetadata.swift | 6 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 30 ++++----- .../SwiftSDKRecipes/SwiftSDKRecipe.swift | 30 ++++++--- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 62 +++++++++++++++---- .../SwiftSDKRecipes/LinuxRecipeTests.swift | 6 +- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 29 ++++++++- 9 files changed, 178 insertions(+), 76 deletions(-) diff --git a/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift index 6904828..ac961ca 100644 --- a/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift +++ b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift @@ -182,4 +182,3 @@ extension Array: CacheKey where Element == FilePath.Component { map(\.string).joined(separator: "\n").hash(with: &hashFunction) } } - diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index e5c9cf6..eecdd99 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -34,7 +34,7 @@ extension Triple.Arch { } extension SwiftSDKGenerator { - public func run(recipe: SwiftSDKRecipe) async throws { + package func run(recipe: SwiftSDKRecipe) async throws { try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in let httpClientType: HTTPClientProtocol.Type @@ -58,13 +58,29 @@ extension SwiftSDKGenerator { let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe) - try await generateDestinationJSON( - toolsetPath: toolsetJSONPath, - sdkDirPath: swiftSDKProduct.sdkDirPath, - recipe: recipe - ) + var artifacts = try await [ + self.artifactID: generateSwiftSDKMetadata( + toolsetPath: toolsetJSONPath, + sdkDirPath: swiftSDKProduct.sdkDirPath, + recipe: recipe + ) + ] + + if recipe.shouldSupportEmbeddedSwift { + let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe, isForEmbeddedSwift: true) - try await generateArtifactBundleManifest(hostTriples: swiftSDKProduct.hostTriples) + artifacts["\(self.artifactID)-embedded"] = try await generateSwiftSDKMetadata( + toolsetPath: toolsetJSONPath, + sdkDirPath: swiftSDKProduct.sdkDirPath, + recipe: recipe, + isForEmbeddedSwift: true + ) + } + + try await generateArtifactBundleManifest( + hostTriples: swiftSDKProduct.hostTriples, + artifacts: artifacts + ) // Extra spaces added for readability for the user print( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index e35ae01..9fb9db2 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -21,10 +21,12 @@ private let encoder: JSONEncoder = { }() extension SwiftSDKGenerator { - func generateToolsetJSON(recipe: SwiftSDKRecipe) throws -> FilePath { + func generateToolsetJSON(recipe: SwiftSDKRecipe, isForEmbeddedSwift: Bool = false) throws -> FilePath { logger.info("Generating toolset JSON file...") - let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending("toolset.json") + let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending( + "\(isForEmbeddedSwift ? "embedded-" : "")toolset.json" + ) var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath @@ -37,18 +39,27 @@ extension SwiftSDKGenerator { } var toolset = Toolset(rootPath: relativeToolchainBinDir.string) - recipe.applyPlatformOptions(toolset: &toolset, targetTriple: self.targetTriple) + recipe.applyPlatformOptions( + toolset: &toolset, + targetTriple: self.targetTriple, + isForEmbeddedSwift: isForEmbeddedSwift + ) try writeFile(at: toolsetJSONPath, encoder.encode(toolset)) return toolsetJSONPath } - func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath, recipe: SwiftSDKRecipe) - throws - { - logger.info("Generating destination JSON file...") + func generateSwiftSDKMetadata( + toolsetPath: FilePath, + sdkDirPath: FilePath, + recipe: SwiftSDKRecipe, + isForEmbeddedSwift: Bool = false + ) throws -> FilePath { + logger.info("Generating Swift SDK metadata JSON file...") - let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending("swift-sdk.json") + let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending( + "\(isForEmbeddedSwift ? "embedded-" : "")swift-sdk.json" + ) var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath var relativeSDKDir = sdkDirPath @@ -67,30 +78,31 @@ extension SwiftSDKGenerator { ) } - var metadata = SwiftSDKMetadataV4.TripleProperties( - sdkRootPath: relativeSDKDir.string, - toolsetPaths: [relativeToolsetPath.string] + var metadata = SwiftSDKMetadataV4( + targetTriples: [ + self.targetTriple.triple: .init( + sdkRootPath: relativeSDKDir.string, + toolsetPaths: [relativeToolsetPath.string] + ) + ] ) recipe.applyPlatformOptions( metadata: &metadata, paths: pathsConfiguration, - targetTriple: self.targetTriple + targetTriple: self.targetTriple, + isForEmbeddedSwift: isForEmbeddedSwift ) try writeFile( at: destinationJSONPath, - encoder.encode( - SwiftSDKMetadataV4( - targetTriples: [ - self.targetTriple.triple: metadata - ] - ) - ) + encoder.encode(metadata) ) + + return destinationJSONPath } - func generateArtifactBundleManifest(hostTriples: [Triple]?) throws { + func generateArtifactBundleManifest(hostTriples: [Triple]?, artifacts: [String: FilePath]) throws { logger.info("Generating .artifactbundle info JSON file...") let artifactBundleManifestPath = pathsConfiguration.artifactBundlePath.appending("info.json") @@ -100,18 +112,18 @@ extension SwiftSDKGenerator { encoder.encode( ArtifactsArchiveMetadata( schemaVersion: "1.0", - artifacts: [ - artifactID: .init( + artifacts: artifacts.mapValues { + .init( type: .swiftSDK, version: self.bundleVersion, variants: [ .init( - path: FilePath(artifactID).appending(self.targetTriple.triple).string, + path: $0.string, supportedTriples: hostTriples.map { $0.map(\.triple) } ) ] ) - ] + } ) ) ) diff --git a/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift b/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift index b693711..ad855ea 100644 --- a/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift +++ b/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift @@ -73,8 +73,8 @@ struct DestinationV3: Encodable { } /// Represents v4 schema of `swift-sdk.json` (previously `destination.json`) files used for cross-compilation. -public struct SwiftSDKMetadataV4: Encodable { - public struct TripleProperties: Encodable { +package struct SwiftSDKMetadataV4: Encodable { + package struct TripleProperties: Encodable { /// Path relative to `swift-sdk.json` containing SDK root. var sdkRootPath: String @@ -98,5 +98,5 @@ public struct SwiftSDKMetadataV4: Encodable { let schemaVersion = "4.0" /// Mapping of triple strings to corresponding properties of such target triple. - let targetTriples: [String: TripleProperties] + var targetTriples: [String: TripleProperties] } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 9c767d7..af3ecc4 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -16,14 +16,14 @@ import Logging import struct SystemPackage.FilePath -public struct LinuxRecipe: SwiftSDKRecipe { - public enum TargetSwiftSource: Sendable { +package struct LinuxRecipe: SwiftSDKRecipe { + package enum TargetSwiftSource: Sendable { case docker(baseSwiftDockerImage: String) case localPackage(FilePath) case remoteTarball } - public enum HostSwiftSource: Sendable, Equatable { + package enum HostSwiftSource: Sendable, Equatable { case localPackage(FilePath) case remoteTarball case preinstalled @@ -35,7 +35,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { let targetSwiftSource: TargetSwiftSource let hostSwiftSource: HostSwiftSource let versionsConfiguration: VersionsConfiguration - public let logger: Logger + package let logger: Logger var shouldUseDocker: Bool { if case .docker = self.targetSwiftSource { @@ -44,7 +44,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { return false } - public init( + package init( targetTriple: Triple, hostTriple: Triple, linuxDistribution: LinuxDistribution, @@ -98,7 +98,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) } - public init( + package init( mainTargetTriple: Triple, mainHostTriple: Triple, linuxDistribution: LinuxDistribution, @@ -116,7 +116,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { self.logger = logger } - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) { + package func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple, isForEmbeddedSwift: Bool) { if self.hostSwiftSource == .preinstalled { toolset.rootPath = nil } @@ -146,20 +146,22 @@ public struct LinuxRecipe: SwiftSDKRecipe { toolset.librarian = Toolset.ToolProperties(path: "llvm-ar") } - public func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + package func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) { var relativeSDKDir = self.sdkDirPath(paths: paths) guard relativeSDKDir.removePrefix(paths.swiftSDKRootPath) else { fatalError("The SDK directory path must be a subdirectory of the Swift SDK root path.") } - metadata.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string - metadata.swiftStaticResourcesPath = relativeSDKDir.appending("usr/lib/swift_static").string + metadata.targetTriples[targetTriple.triple]?.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string + metadata.targetTriples[targetTriple.triple]?.swiftStaticResourcesPath = + relativeSDKDir.appending("usr/lib/swift_static").string } - public var defaultArtifactID: String { + package var defaultArtifactID: String { """ \(self.versionsConfiguration.swiftVersion)_\(self.linuxDistribution.name.rawValue)_\( self.linuxDistribution @@ -219,7 +221,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { return [self.mainHostTriple] } - public func makeSwiftSDK( + package func makeSwiftSDK( generator: SwiftSDKGenerator, engine: QueryEngine, httpClient client: some HTTPClientProtocol diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 5161519..6b11451 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -15,23 +15,25 @@ import Logging import struct SystemPackage.FilePath -public struct SwiftSDKProduct { +package struct SwiftSDKProduct { let sdkDirPath: FilePath /// Array of supported host triples. `nil` indicates the SDK can be universally used. let hostTriples: [Triple]? } /// A protocol describing a set of platform specific instructions to make a Swift SDK -public protocol SwiftSDKRecipe: Sendable { +package protocol SwiftSDKRecipe: Sendable { /// Update the given toolset with platform specific options func applyPlatformOptions( toolset: inout Toolset, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) /// The default identifier of the Swift SDK @@ -45,15 +47,23 @@ public protocol SwiftSDKRecipe: Sendable { generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol - ) async throws - -> SwiftSDKProduct + ) async throws -> SwiftSDKProduct + + var shouldSupportEmbeddedSwift: Bool { get } } extension SwiftSDKRecipe { - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) {} - public func applyPlatformOptions( + package func applyPlatformOptions( + toolset: inout Toolset, + targetTriple: Triple, + isForEmbeddedSwift: Bool + ) {} + package func applyPlatformOptions( metadata: inout SwiftSDKMetadataV4.TripleProperties, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) {} + + package var shouldSupportEmbeddedSwift: Bool { false } } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index f3b16dc..de9a8d9 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -15,24 +15,24 @@ import Logging import struct SystemPackage.FilePath -public struct WebAssemblyRecipe: SwiftSDKRecipe { +package struct WebAssemblyRecipe: SwiftSDKRecipe { let hostSwiftPackage: HostToolchainPackage? let targetSwiftPackagePath: FilePath let wasiSysroot: FilePath let swiftVersion: String - public let logger: Logger + package let logger: Logger - public struct HostToolchainPackage: Sendable { + package struct HostToolchainPackage: Sendable { let path: FilePath let triple: Triple - public init(path: FilePath, triple: Triple) { + package init(path: FilePath, triple: Triple) { self.path = path self.triple = triple } } - public init( + package init( hostSwiftPackage: HostToolchainPackage?, targetSwiftPackagePath: FilePath, wasiSysroot: FilePath, @@ -46,13 +46,35 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { self.logger = logger } - public var defaultArtifactID: String { + package var defaultArtifactID: String { "\(self.swiftVersion)_wasm" } - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) { + package let shouldSupportEmbeddedSwift = true + + package func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple, isForEmbeddedSwift: Bool) { // We only support static linking for WebAssembly for now, so make it the default. toolset.swiftCompiler = Toolset.ToolProperties(extraCLIOptions: ["-static-stdlib"]) + + if isForEmbeddedSwift { + let ccOptions = ["-D__EMBEDDED_SWIFT__"] + toolset.cCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) + toolset.cxxCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) + + toolset.swiftCompiler?.extraCLIOptions?.append( + contentsOf: [ + "-enable-experimental-feature", "Embedded", "-wmo", + ] + ) + + toolset.swiftCompiler?.extraCLIOptions?.append( + // libraries required for concurrency + contentsOf: ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { + ["-Xlinker", $0] + } + ) + } + if targetTriple.environmentName == "threads" { // Enable features required for threading support let ccOptions = [ @@ -82,10 +104,11 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } } - public func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + package func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) { var relativeToolchainDir = paths.toolchainDirPath guard relativeToolchainDir.removePrefix(paths.swiftSDKRootPath) else { @@ -93,12 +116,25 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { "The toolchain bin directory path must be a subdirectory of the Swift SDK root path." ) } - metadata.swiftStaticResourcesPath = + + var tripleProperties = metadata.targetTriples[targetTriple.triple]! + tripleProperties.swiftStaticResourcesPath = relativeToolchainDir.appending("usr/lib/swift_static").string - metadata.swiftResourcesPath = metadata.swiftStaticResourcesPath + tripleProperties.swiftResourcesPath = + isForEmbeddedSwift + ? relativeToolchainDir.appending("usr/lib/swift").string + : tripleProperties.swiftStaticResourcesPath + + var finalTriple = targetTriple + if isForEmbeddedSwift { + metadata.targetTriples.removeValue(forKey: targetTriple.triple) + finalTriple = Triple("wasm32-unknown-wasip1") + } + + metadata.targetTriples[finalTriple.triple] = tripleProperties } - public func makeSwiftSDK( + package func makeSwiftSDK( generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index ddda1de..b57807e 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -85,7 +85,8 @@ final class LinuxRecipeTests: XCTestCase { var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: testCase.targetTriple + targetTriple: testCase.targetTriple, + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, testCase.expectedSwiftCompilerOptions) XCTAssertEqual(toolset.linker?.path, testCase.expectedLinkerPath) @@ -103,7 +104,8 @@ final class LinuxRecipeTests: XCTestCase { var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("x86_64-unknown-linux-gnu") + targetTriple: Triple("x86_64-unknown-linux-gnu"), + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.rootPath, nil) XCTAssertEqual( diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift index 228f4f5..dff9499 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -33,7 +33,8 @@ final class WebAssemblyRecipeTests: XCTestCase { var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("wasm32-unknown-wasi") + targetTriple: Triple("wasm32-unknown-wasi"), + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, ["-static-stdlib"]) XCTAssertNil(toolset.cCompiler) @@ -41,12 +42,36 @@ final class WebAssemblyRecipeTests: XCTestCase { XCTAssertNil(toolset.linker) } + func testEmbeddedToolOptions() { + let recipe = self.createRecipe() + var toolset = Toolset(rootPath: nil) + recipe.applyPlatformOptions( + toolset: &toolset, + targetTriple: Triple("wasm32-unknown-wasi"), + isForEmbeddedSwift: true + ) + XCTAssertEqual( + toolset.swiftCompiler?.extraCLIOptions, + [ + "-static-stdlib", + "-enable-experimental-feature", "Embedded", "-wmo", + ] + + ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { + ["-Xlinker", $0] + } + ) + XCTAssertEqual(toolset.cCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) + XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) + XCTAssertNil(toolset.linker) + } + func testToolOptionsWithThreads() { let recipe = self.createRecipe() var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("wasm32-unknown-wasip1-threads") + targetTriple: Triple("wasm32-unknown-wasip1-threads"), + isForEmbeddedSwift: false ) XCTAssertEqual( toolset.swiftCompiler?.extraCLIOptions, From d45075e656908493db22405c4bff42f0e53c0e7e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 16 May 2025 12:44:38 +0100 Subject: [PATCH 14/17] Fix use of absolute paths to Swift SDK metatadata (#212) The generator incorrectly recorded absolute paths to Swift SDK metadata in bundle archive metadata, which is fragile and also can't be handled by SwiftPM. --- .../SwiftSDKGenerator+Metadata.swift | 8 +++- .../SwiftSDKGenerator+MetadataTests.swift | 45 ++++++++++++++++--- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index 9fb9db2..fbb056b 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -113,12 +113,16 @@ extension SwiftSDKGenerator { ArtifactsArchiveMetadata( schemaVersion: "1.0", artifacts: artifacts.mapValues { - .init( + var relativePath = $0 + let prefixRemoved = relativePath.removePrefix(pathsConfiguration.artifactBundlePath) + assert(prefixRemoved) + + return .init( type: .swiftSDK, version: self.bundleVersion, variants: [ .init( - path: $0.string, + path: relativePath.string, supportedTriples: hostTriples.map { $0.map(\.triple) } ) ] diff --git a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift index b599075..5074efd 100644 --- a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift +++ b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift @@ -16,6 +16,12 @@ import XCTest @testable import SwiftSDKGenerator +#if canImport(FoundationEssentials) + import FoundationEssentials +#else + import Foundation +#endif + final class SwiftSDKGeneratorMetadataTests: XCTestCase { let logger = Logger(label: "SwiftSDKGeneratorMetadataTests") @@ -54,19 +60,44 @@ final class SwiftSDKGeneratorMetadataTests: XCTestCase { // Make sure the file exists let sdkSettingsFile = sdkDirPath.appending("SDKSettings.json") - let fileExists = await sdk.doesFileExist(at: sdkSettingsFile) + var fileExists = await sdk.doesFileExist(at: sdkSettingsFile) XCTAssertTrue(fileExists) // Read back file, make sure it contains the expected data - let data = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) - XCTAssertNotNil(data) - XCTAssertTrue(data!.contains(testCase.bundleVersion)) - XCTAssertTrue(data!.contains("(\(testCase.targetTriple.archName))")) - XCTAssertTrue(data!.contains(linuxDistribution.description)) - XCTAssertTrue(data!.contains(testCase.expectedCanonicalName)) + let maybeString = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) + let string = try XCTUnwrap(maybeString) + XCTAssertTrue(string.contains(testCase.bundleVersion)) + XCTAssertTrue(string.contains("(\(testCase.targetTriple.archName))")) + XCTAssertTrue(string.contains(linuxDistribution.description)) + XCTAssertTrue(string.contains(testCase.expectedCanonicalName)) // Cleanup try await sdk.removeFile(at: sdkSettingsFile) + + try await sdk.createDirectoryIfNeeded(at: sdk.pathsConfiguration.artifactBundlePath) + + // Generate bundle metadata + try await sdk.generateArtifactBundleManifest( + hostTriples: [sdk.targetTriple], + artifacts: ["foo": sdk.pathsConfiguration.artifactBundlePath.appending("bar.json")] + ) + + // Make sure the file exists + let archiveMetadataFile = await sdk.pathsConfiguration.artifactBundlePath.appending("info.json") + fileExists = await sdk.doesFileExist(at: archiveMetadataFile) + XCTAssertTrue(fileExists) + + // Read back file, make sure it contains the expected data + let data = try await sdk.readFile(at: archiveMetadataFile) + let decodedMetadata = try JSONDecoder().decode(ArtifactsArchiveMetadata.self, from: data) + XCTAssertEqual(decodedMetadata.artifacts.count, 1) + for (id, artifact) in decodedMetadata.artifacts { + XCTAssertEqual(id, "foo") + XCTAssertEqual(artifact.variants, [.init(path: "bar.json", supportedTriples: [testCase.targetTriple.triple])]) + } + + // Cleanup + try await sdk.removeFile(at: archiveMetadataFile) } } } From d7264b74bbb5cb16e30e9428eb58b9996b32a42b Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 16 May 2025 15:17:52 +0100 Subject: [PATCH 15/17] Symlink `libclang_rt.builtins.a` for Embedded Swift for WASI (#213) Embedded Swift looks up Clang's compiler-rt static library in a different path than non-embedded. These paths could be normalized in the future, but for now the easiest fix seems to be just to symlink the directory the static library is located in. --- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index de9a8d9..8497f9b 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -181,7 +181,7 @@ package struct WebAssemblyRecipe: SwiftSDKRecipe { ) } - let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending( + let autolinkExtractPath = pathsConfiguration.toolchainBinDirPath.appending( "swift-autolink-extract" ) @@ -195,6 +195,13 @@ package struct WebAssemblyRecipe: SwiftSDKRecipe { try await generator.createSymlink(at: autolinkExtractPath, pointingTo: "swift") } + // Embedded Swift looks up clang compiler-rt in a different path. + let embeddedCompilerRTPath = pathsConfiguration.toolchainDirPath.appending( + "usr/lib/swift/clang/lib/wasip1" + ) + + try await generator.createSymlink(at: embeddedCompilerRTPath, pointingTo: "wasi") + // Copy the WASI sysroot into the SDK bundle. let sdkDirPath = pathsConfiguration.swiftSDKRootPath.appending("WASI.sdk") try await generator.rsyncContents(from: self.wasiSysroot, to: sdkDirPath) From 8131942e68a9bc6a26f914c165d9abb2ad372bbc Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sat, 17 May 2025 09:01:08 -0400 Subject: [PATCH 16/17] Add conditional experimental- prefix to completion message with anything less than 6.0 --- .../Generator/SwiftSDKGenerator+Entrypoint.swift | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index eecdd99..3e8d550 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -82,15 +82,21 @@ extension SwiftSDKGenerator { artifacts: artifacts ) + #if compiler(>=6.0) + let sdkCommandPrefix = "" + #else + let sdkCommandPrefix = "experimental-" + #endif + // Extra spaces added for readability for the user print( """ All done! Install the newly generated SDK with this command: - swift experimental-sdk install \(pathsConfiguration.artifactBundlePath) + swift \(sdkCommandPrefix)sdk install \(pathsConfiguration.artifactBundlePath) After that, use the newly installed SDK when building with this command: - swift build --experimental-swift-sdk \(artifactID) + swift build --\(sdkCommandPrefix)swift-sdk \(artifactID) """ ) From 16fea5006732fc1e8ea7e95387658d28beb0a59a Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sat, 17 May 2025 09:08:33 -0400 Subject: [PATCH 17/17] Add more info to distribution section on how to generate checksums for artifactbundles to be distributed --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 4599197..0feed34 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,19 @@ The `.artifactbundle` directory produced in the previous section can be packaged in this form. Users of such Swift SDK bundle archive can easily install it with `swift sdk install` command, which supports both local file system paths and public `http://` and `https://` URLs as an argument. +To make an `.artifactbundle.tar.gz` archive installable directly from `http://` and `https://` URLs, a checksum must be +generated. This can be done using the `swift package compute-checksum` command, like this: + +```console +swift package compute-checksum ./Bundles/6.1-RELEASE_ubuntu_jammy_x86_64.artifactbundle.tar.gz + +``` + +This checksum should be provided along with the artifact bundles to be included with the sdk install invocation: + +```console +swift sdk install https://my-public-site/sdks/6.1-RELEASE_ubuntu_jammy_x86_64.artifactbundle.tar.gz --checksum +``` ## Contributing