From f9a14b8f10ff7031c19b3eabd0971979121c45a5 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sat, 30 Mar 2024 17:33:08 +0000 Subject: [PATCH 1/3] build: Build Swift SDK for WebAssembly This change adds a new product, WasmSwiftSDK, to build the Swift SDK for WebAssembly. The product is built using the swift-sdk-generator package and takes just built WebAssembly stdlib, clang runtime libraries, and wasi-sysroot as input, and produces a Swift SDK artifactbundle under swift-sdk-generator/Bundles. --- .../build_script_invocation.py | 2 + .../swift_build_support/products/__init__.py | 2 + .../products/wasmstdlib.py | 1 + .../products/wasmswiftsdk.py | 116 ++++++++++++++++++ .../update-checkout-config.json | 3 + 5 files changed, 124 insertions(+) create mode 100644 utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index cb88b9b2e1529..5637668271b9a 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -695,6 +695,8 @@ def compute_product_pipelines(self): is_enabled=self.args.build_wasmstdlib) builder.add_product(products.WasmThreadsStdlib, is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.WasmSwiftSDK, + is_enabled=self.args.build_wasmstdlib) # Keep SwiftDriver at last. # swift-driver's integration with the build scripts is not fully diff --git a/utils/swift_build_support/swift_build_support/products/__init__.py b/utils/swift_build_support/swift_build_support/products/__init__.py index 51dc3ba018d9b..1664e20e55f47 100644 --- a/utils/swift_build_support/swift_build_support/products/__init__.py +++ b/utils/swift_build_support/swift_build_support/products/__init__.py @@ -40,6 +40,7 @@ from .wasisysroot import WASILibc, WasmLLVMRuntimeLibs, WasmThreadsLLVMRuntimeLibs from .wasmkit import WasmKit from .wasmstdlib import WasmStdlib, WasmThreadsStdlib +from .wasmswiftsdk import WasmSwiftSDK from .xctest import XCTest from .zlib import Zlib @@ -79,4 +80,5 @@ 'WasmStdlib', 'WasmThreadsLLVMRuntimeLibs', 'WasmThreadsStdlib', + 'WasmSwiftSDK', ] diff --git a/utils/swift_build_support/swift_build_support/products/wasmstdlib.py b/utils/swift_build_support/swift_build_support/products/wasmstdlib.py index 1d5f4a78df227..dabdff759c091 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmstdlib.py +++ b/utils/swift_build_support/swift_build_support/products/wasmstdlib.py @@ -42,6 +42,7 @@ def build(self, host_target): self._build(host_target, 'wasm32-wasi') def _build(self, host_target, target_triple): + self.cmake_options.define('CMAKE_INSTALL_PREFIX:PATH', '/usr') self.cmake_options.define('CMAKE_BUILD_TYPE:STRING', self._build_variant) self.cmake_options.define( 'SWIFT_STDLIB_BUILD_TYPE:STRING', self._build_variant) diff --git a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py new file mode 100644 index 0000000000000..dc2eb3aaded11 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py @@ -0,0 +1,116 @@ +# swift_build_support/products/wasmswiftsdk.py ------------------*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2024 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- + +import os + +from . import product +from . import wasisysroot +from .wasmstdlib import WasmStdlib, WasmThreadsStdlib +from .. import shell + + +class WasmSwiftSDK(product.Product): + @classmethod + def product_source_name(cls): + return "swift-sdk-generator" + + @classmethod + def is_build_script_impl_product(cls): + return False + + @classmethod + def is_before_build_script_impl_product(cls): + return False + + def should_build(self, host_target): + return self.args.build_wasmstdlib + + def should_test(self, host_target): + return False + + def _target_package_path(self, target_triple): + return os.path.join(self.build_dir, 'Toolchains', target_triple) + + def _build_target_package(self, target_triple, + stdlib_build_path, llvm_runtime_libs_build_path): + + dest_dir = self._target_package_path(target_triple) + shell.rmtree(dest_dir) + shell.makedirs(dest_dir) + + # Build toolchain package for standalone stdlib + with shell.pushd(stdlib_build_path): + shell.call([self.toolchain.cmake, '--install', '.'], + env={'DESTDIR': dest_dir}) + + # Copy clang builtin libraries + with shell.pushd(llvm_runtime_libs_build_path): + for dirname in ['clang', 'swift/clang', 'swift_static/clang']: + clang_dir = os.path.join(dest_dir, f'usr/lib/{dirname}') + shell.call([self.toolchain.cmake, '--install', '.', + '--component', 'clang_rt.builtins-wasm32'], + env={'DESTDIR': clang_dir}) + + return dest_dir + + def build(self, host_target): + build_root = os.path.dirname(self.build_dir) + llvm_runtime_libs_build_path = os.path.join( + build_root, '%s-%s' % ('wasmllvmruntimelibs', host_target)) + + target_packages = [] + for target_triple, short_triple, build_basename in [ + ('wasm32-unknown-wasi', 'wasm32-wasi', 'wasmstdlib'), + # TODO: Enable threads stdlib once sdk-generator supports multi-target SDK + # ('wasm32-unknown-wasip1-threads', 'wasmthreadsstdlib'), + ]: + stdlib_build_path = os.path.join( + build_root, '%s-%s' % (build_basename, host_target)) + package_path = self._build_target_package( + target_triple, stdlib_build_path, llvm_runtime_libs_build_path) + target_packages.append((target_triple, package_path)) + + swiftc_path = os.path.abspath(self.toolchain.swiftc) + toolchain_path = os.path.dirname(os.path.dirname(swiftc_path)) + swift_run = os.path.join(toolchain_path, 'bin', 'swift-run') + + swift_version = os.environ.get('TOOLCHAIN_VERSION', + 'swift-DEVELOPMENT-SNAPSHOT').lstrip('swift-') + run_args = [ + swift_run, + '--package-path', self.source_dir, + '--build-path', self.build_dir, + 'swift-sdk-generator', + 'make-wasm-sdk', + '--swift-version', swift_version, + ] + for target_triple, package_path in target_packages: + run_args.extend(['--target', target_triple]) + run_args.extend(['--target-swift-package-path', package_path]) + wasi_sysroot = wasisysroot.WASILibc.sysroot_install_path( + build_root, short_triple) + run_args.extend(['--wasi-sysroot', wasi_sysroot]) + + env = dict(os.environ) + env['SWIFTCI_USE_LOCAL_DEPS'] = '1' + + shell.call(run_args, env=env) + + def test(self, host_target): + pass + + def should_install(self, host_target): + return False + + @classmethod + def get_dependencies(cls): + return [WasmStdlib, WasmThreadsStdlib] diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 88d4e00bd9d11..9863d54146b8a 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -83,6 +83,8 @@ "remote": { "id": "apple/swift-nio-ssl" } }, "swift-experimental-string-processing": { "remote": { "id": "apple/swift-experimental-string-processing" } }, + "swift-sdk-generator": { + "remote": { "id": "apple/swift-sdk-generator" } }, "swift-llvm-bindings": { "remote": { "id": "apple/swift-llvm-bindings" } }, "llvm-project": { @@ -149,6 +151,7 @@ "swift-markdown": "main", "swift-nio": "2.31.2", "swift-experimental-string-processing": "swift/main", + "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-20", "wasmkit": "0.0.3", "curl": "curl-8_5_0", From 4e4047dead1c6924ada7222cbb8ceaf19fa8b719 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Tue, 23 Apr 2024 16:19:59 +0000 Subject: [PATCH 2/3] update-checkout: Update swift-sdk-generator dependencies swift-nio and swift-async-algorithms have been updated to support `SWIFTCI_USE_LOCAL_DEPS` convention. This change updates them to the versions including the support. --- utils/update_checkout/update-checkout-config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 9863d54146b8a..47f48a371c014 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -125,7 +125,7 @@ "swift-crypto": "3.0.0", "swift-certificates": "1.0.1", "swift-asn1": "1.0.0", - "swift-async-algorithms": "1.0.0", + "swift-async-algorithms": "main", "swift-driver": "main", "swift-log": "1.5.4", "swift-numerics": "1.0.2", @@ -149,7 +149,7 @@ "swift-docc-render-artifact": "main", "swift-docc-symbolkit": "main", "swift-markdown": "main", - "swift-nio": "2.31.2", + "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-20", From 1148e26ae40b963274a61c1187f2101b2cb2ad63 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Tue, 23 Apr 2024 16:25:13 +0000 Subject: [PATCH 3/3] build: Build Swift SDK earlier than SwiftPM to allow SwiftPM to use it Just built Swift SDK would be useful for SwiftPM to perform integration tests. All of those products do not depend on just built SwiftPM (WasmKit is built using host tools), so it is safe to build them earlier. --- .../build_script_invocation.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index 5637668271b9a..68eff1377d71d 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -657,6 +657,21 @@ def compute_product_pipelines(self): # Begin the post build-script-impl build phase. builder.begin_pipeline() + builder.add_product(products.WASILibc, + is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.WasmLLVMRuntimeLibs, + is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.WasmThreadsLLVMRuntimeLibs, + is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.WasmKit, + is_enabled=self.args.build_wasmkit) + builder.add_product(products.WasmStdlib, + is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.WasmThreadsStdlib, + is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.WasmSwiftSDK, + is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.SwiftPM, is_enabled=self.args.build_swiftpm) builder.add_product(products.SwiftSyntax, @@ -683,20 +698,6 @@ def compute_product_pipelines(self): is_enabled=self.args.install_swiftdocc) builder.add_product(products.MinimalStdlib, is_enabled=self.args.build_minimalstdlib) - builder.add_product(products.WASILibc, - is_enabled=self.args.build_wasmstdlib) - builder.add_product(products.WasmLLVMRuntimeLibs, - is_enabled=self.args.build_wasmstdlib) - builder.add_product(products.WasmThreadsLLVMRuntimeLibs, - is_enabled=self.args.build_wasmstdlib) - builder.add_product(products.WasmKit, - is_enabled=self.args.build_wasmkit) - builder.add_product(products.WasmStdlib, - is_enabled=self.args.build_wasmstdlib) - builder.add_product(products.WasmThreadsStdlib, - is_enabled=self.args.build_wasmstdlib) - builder.add_product(products.WasmSwiftSDK, - is_enabled=self.args.build_wasmstdlib) # Keep SwiftDriver at last. # swift-driver's integration with the build scripts is not fully