Main merge, more ideas for experimental README changes

.github/workflows/pull_request.yml

@@ -0,0 +1,18 @@
+name: Pull request
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  tests:
+    name: Test
+    uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
+    with:
+      linux_pre_build_command: apt-get update && apt-get install -y locales locales-all libsqlite3-dev
+      enable_windows_checks: false
+  soundness:
+    name: Soundness
+    uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main
+    with:
+      api_breakage_check_enabled: false

.license_header_template

@@ -0,0 +1,12 @@
+@@===----------------------------------------------------------------------===@@
+@@
+@@ This source file is part of the Swift open source project
+@@
+@@ Copyright (c) YEARS Apple Inc. and the Swift project authors
+@@ Licensed under Apache License v2.0 with Runtime Library Exception
+@@
+@@ See https://swift.org/LICENSE.txt for license information
+@@ See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+@@
+@@===----------------------------------------------------------------------===@@
+

.licenseignore

@@ -0,0 +1,22 @@
+.gitignore
+.swiftformat
+.swiftformatignore
+.editorconfig
+.unacceptablelanguageignore
+Brewfile
+Package.swift
+Package.resolved
+*.md
+*.txt
+*.yml
+**/.editorconfig
+**/*.docc/**
+**/*.entitlements
+**/*.input
+**/*.modulemap
+**/*.plist
+**/*.xcodeproj/**
+**/CODEOWNERS
+**/Dockerfile
+**/Package.swift
+Utilities/git.commit.template

.swift-format

@@ -0,0 +1,70 @@
+{
+  "fileScopedDeclarationPrivacy" : {
+    "accessLevel" : "private"
+  },
+  "indentConditionalCompilationBlocks" : true,
+  "indentSwitchCaseLabels" : false,
+  "indentation" : {
+    "spaces" : 2
+  },
+  "lineBreakAroundMultilineExpressionChainComponents" : false,
+  "lineBreakBeforeControlFlowKeywords" : false,
+  "lineBreakBeforeEachArgument" : true,
+  "lineBreakBeforeEachGenericRequirement" : false,
+  "lineLength" : 120,
+  "maximumBlankLines" : 1,
+  "multiElementCollectionTrailingCommas" : true,
+  "noAssignmentInExpressions" : {
+    "allowedFunctions" : [
+      "XCTAssertNoThrow"
+    ]
+  },
+  "prioritizeKeepingFunctionOutputTogether" : false,
+  "respectsExistingLineBreaks" : true,
+  "rules" : {
+    "AllPublicDeclarationsHaveDocumentation" : false,
+    "AlwaysUseLiteralForEmptyCollectionInit" : false,
+    "AlwaysUseLowerCamelCase" : false,
+    "AmbiguousTrailingClosureOverload" : true,
+    "BeginDocumentationCommentWithOneLineSummary" : false,
+    "DoNotUseSemicolons" : true,
+    "DontRepeatTypeInStaticProperties" : true,
+    "FileScopedDeclarationPrivacy" : true,
+    "FullyIndirectEnum" : true,
+    "GroupNumericLiterals" : true,
+    "IdentifiersMustBeASCII" : true,
+    "NeverForceUnwrap" : false,
+    "NeverUseForceTry" : false,
+    "NeverUseImplicitlyUnwrappedOptionals" : false,
+    "NoAccessLevelOnExtensionDeclaration" : true,
+    "NoAssignmentInExpressions" : true,
+    "NoBlockComments" : true,
+    "NoCasesWithOnlyFallthrough" : true,
+    "NoEmptyTrailingClosureParentheses" : true,
+    "NoLabelsInCasePatterns" : true,
+    "NoLeadingUnderscores" : false,
+    "NoParensAroundConditions" : true,
+    "NoPlaygroundLiterals" : true,
+    "NoVoidReturnOnFunctionSignature" : true,
+    "OmitExplicitReturns" : false,
+    "OneCasePerLine" : true,
+    "OneVariableDeclarationPerLine" : true,
+    "OnlyOneTrailingClosureArgument" : true,
+    "OrderedImports" : true,
+    "ReplaceForEachWithForLoop" : true,
+    "ReturnVoidInsteadOfEmptyTuple" : true,
+    "TypeNamesShouldBeCapitalized" : true,
+    "UseEarlyExits" : false,
+    "UseExplicitNilCheckInConditions" : true,
+    "UseLetInEveryBoundCaseVariable" : false,
+    "UseShorthandTypeNames" : true,
+    "UseSingleLinePropertyGetter" : true,
+    "UseSynthesizedInitializer" : true,
+    "UseTripleSlashForDocumentationComments" : true,
+    "UseWhereClausesInForLoops" : false,
+    "ValidateDocumentationComments" : false
+  },
+  "spacesAroundRangeFormationOperators" : false,
+  "tabWidth" : 8,
+  "version" : 1
+}

.swiftformatignore

@@ -0,0 +1,2 @@
+Sources/Helpers/Vendor/*
+Sources/AsyncProcess/ProcessExecutor.swift

.unacceptablelanguageignore

@@ -0,0 +1,2 @@
+Sources/AsyncProcess/ProcessExecutor.swift
+Tests/AsyncProcessTests/IntegrationTests.swift

CONTRIBUTING.md

@@ -58,29 +58,6 @@ We require that your commit messages match our template. The easiest way to do t
 
     git config commit.template Utilities/git.commit.template
 
-### Run `./Utilities/soundness.sh`
-
-The scripts directory contains a [`soundness.sh` script](https://github.com/apple/swift-sdk-generator/blob/main/Utilities/soundness.sh) 
-that enforces additional checks, like license headers and formatting style.
-
-Please make sure to run `./Utilities/soundness.sh` before pushing a change upstream, otherwise it is likely the PR validation will fail
-on minor changes such as formatting issues.
-
-For frequent contributors, we recommend adding the script as a [git pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks), which you can do via executing the following command in the project root directory: 
-
-```bash
-cat << EOF > .git/hooks/pre-push
-
-if [[ -f "Utilities/soundness.sh" ]]; then
-  Utilities/soundness.sh
-fi
-EOF
-```
-
-Which makes the script execute, and only allow the `git push` to complete if the check has passed.
-
-In the case of formatting issues, you can then `git add` the formatting changes, and attempt the push again. 
-
 ## How to contribute your work
 
-Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review.
+Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review.

Docker/docker-compose.yaml

@@ -19,10 +19,6 @@ services:
       - ..:/code:z
     working_dir: /code
 
-  soundness:
-    <<: *common
-    command: /bin/bash -xcl "swift -version && uname -a && ./Utilities/soundness.sh"
-
   test:
     <<: *common
     environment:

Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version: 5.8
+// swift-tools-version: 5.9
 // The swift-tools-version declares the minimum version of Swift required to build this package.
 
 import PackageDescription
@@ -11,7 +11,7 @@ let package = Package(
     .executable(
       name: "swift-sdk-generator",
       targets: ["GeneratorCLI"]
-    ),
+    )
   ],
   targets: [
     // Targets are the basic building blocks of a package. A target can define a module or a test suite.
@@ -23,7 +23,7 @@ let package = Package(
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
       ],
       swiftSettings: [
-        .enableExperimentalFeature("StrictConcurrency=complete"),
+        .enableExperimentalFeature("StrictConcurrency=complete")
       ]
     ),
     .target(
@@ -38,22 +38,22 @@ let package = Package(
       ],
       exclude: ["Dockerfiles"],
       swiftSettings: [
-        .enableExperimentalFeature("StrictConcurrency=complete"),
+        .enableExperimentalFeature("StrictConcurrency=complete")
       ]
     ),
     .testTarget(
       name: "SwiftSDKGeneratorTests",
       dependencies: [
-        "SwiftSDKGenerator",
+        "SwiftSDKGenerator"
       ],
       swiftSettings: [
-        .enableExperimentalFeature("StrictConcurrency=complete"),
+        .enableExperimentalFeature("StrictConcurrency=complete")
       ]
     ),
     .testTarget(
       name: "GeneratorEngineTests",
       dependencies: [
-        "Helpers",
+        "Helpers"
       ]
     ),
     .target(
@@ -66,13 +66,13 @@ let package = Package(
       ],
       exclude: ["Vendor/README.md"],
       swiftSettings: [
-        .enableExperimentalFeature("StrictConcurrency=complete"),
+        .enableExperimentalFeature("StrictConcurrency=complete")
       ]
     ),
     .testTarget(
       name: "HelpersTests",
       dependencies: [
-        "Helpers",
+        "Helpers"
       ]
     ),
     .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"),

README.md

@@ -54,14 +54,14 @@ The generator also allows cross-compiling between any Linux distributions offici
 | macOS (arm64)  | ✅ macOS 13.0+            | ❌                         |
 | macOS (x86_64) | ✅ macOS 13.0+[^1]        | ❌                         |
 | Ubuntu         | ✅ 20.04+                 | ✅ 20.04+                  |
-| RHEL           | ✅ Fedora 39[^2], UBI 9   | ✅ UBI 9                   |
+| Debian         | ✅ 11, 12[^2]             | ✅ 11, 12[^2]              |
+| RHEL           | ✅ Fedora 39, UBI 9       | ✅ Fedora 39, UBI 9[^3]    |
 | Amazon Linux 2 | ✅ Supported              | ✅ Supported[^3]           |
-| Debian 12      | ✅ Supported[^2]          | ✅ Supported[^2][^3]       |
 
 [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built
 from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also
 need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`).
-[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms.
+[^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible.
 [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138).
 
 ## How to use it
@@ -94,7 +94,7 @@ subsequently as `<generated_sdk_id>`.
 Create a new project to verify that the SDK works:
 
 ```
-mkdir cross-compilation test
+mkdir cross-compilation-test
 cd cross-compilation-test
 swift package init --type executable
 ```
@@ -185,6 +185,19 @@ The `.artifactbundle` directory produced in the previous section can be packaged
 in this form. Users of such Swift SDK bundle archive can easily install it with `swift sdk install`
 command, which supports both local file system paths and public `http://` and `https://` URLs as an argument.
 
+To make an `.artifactbundle.tar.gz` archive installable directly from `http://` and `https://` URLs, a checksum must be
+generated. This can be done using the `swift package compute-checksum` command, like this:
+
+```console
+swift package compute-checksum ./Bundles/6.1-RELEASE_ubuntu_jammy_x86_64.artifactbundle.tar.gz
+<checksum>
+```
+
+This checksum should be provided along with the artifact bundles to be included with the sdk install invocation:
+
+```console
+swift sdk install https://my-public-site/sdks/6.1-RELEASE_ubuntu_jammy_x86_64.artifactbundle.tar.gz --checksum <checksum>
+```
 
 ## Contributing
 

Sources/AsyncProcess/ChunkSequence.swift

@@ -13,9 +13,9 @@
 import NIO
 
 #if os(Linux) || os(Android) || os(Windows)
-@preconcurrency import Foundation
+  @preconcurrency import Foundation
 #else
-import Foundation
+  import Foundation
 #endif
 
 public struct IllegalStreamConsumptionError: Error {
@@ -53,9 +53,9 @@ public struct ChunkSequence: AsyncSequence & Sendable {
       } else {
         throw IllegalStreamConsumptionError(
           description: """
-          Either `.discard`ed, `.inherit`ed or redirected this stream to a `.fileHandle`,
-          cannot also consume it. To consume, please `.stream` it.
-          """
+            Either `.discard`ed, `.inherit`ed or redirected this stream to a `.fileHandle`,
+            cannot also consume it. To consume, please `.stream` it.
+            """
         )
       }
     }

Sources/AsyncProcess/FileContentStream.swift

@@ -255,7 +255,10 @@ private final class ReadIntoAsyncChannelHandler: ChannelDuplexHandler {
 
 extension FileHandle {
   func fileContentStream(eventLoop: EventLoop) throws -> FileContentStream {
-    let asyncBytes = try FileContentStream(fileDescriptor: self.fileDescriptor, eventLoop: eventLoop)
+    let asyncBytes = try FileContentStream(
+      fileDescriptor: self.fileDescriptor,
+      eventLoop: eventLoop
+    )
     try self.close()
     return asyncBytes
   }
@@ -272,8 +275,8 @@ extension FileContentStream {
   }
 }
 
-public extension AsyncSequence where Element == ByteBuffer, Self: Sendable {
-  func splitIntoLines(
+extension AsyncSequence where Element == ByteBuffer, Self: Sendable {
+  public func splitIntoLines(
     dropTerminator: Bool = true,
     maximumAllowableBufferSize: Int = 1024 * 1024,
     dropLastChunkIfNoNewline: Bool = false
@@ -286,14 +289,13 @@ public extension AsyncSequence where Element == ByteBuffer, Self: Sendable {
     )
   }
 
-  var strings: AsyncMapSequence<Self, String> {
+  public var strings: AsyncMapSequence<Self, String> {
     self.map { String(buffer: $0) }
   }
 }
 
 public struct AsyncByteBufferLineSequence<Base: Sendable>: AsyncSequence & Sendable
-  where Base: AsyncSequence, Base.Element == ByteBuffer
-{
+where Base: AsyncSequence, Base.Element == ByteBuffer {
   public typealias Element = ByteBuffer
   private let underlying: Base
   private let dropTerminator: Bool
@@ -329,7 +331,9 @@ public struct AsyncByteBufferLineSequence<Base: Sendable>: AsyncSequence & Senda
         self.buffer.last?.readableBytesView
       }
 
-      mutating func concatenateEverything(upToLastChunkLengthToConsume lastLength: Int) -> ByteBuffer {
+      mutating func concatenateEverything(upToLastChunkLengthToConsume lastLength: Int)
+        -> ByteBuffer
+      {
         var output = ByteBuffer()
         output.reserveCapacity(lastLength + self.byteCountButLast)
 
@@ -430,7 +434,8 @@ public struct AsyncByteBufferLineSequence<Base: Sendable>: AsyncSequence & Senda
   }
 
   public init(
-    _ underlying: Base, dropTerminator: Bool,
+    _ underlying: Base,
+    dropTerminator: Bool,
     maximumAllowableBufferSize: Int,
     dropLastChunkIfNoNewline: Bool
   ) {