From c86e3af4546d3376d8b79c3eb709c8386d040f09 Mon Sep 17 00:00:00 2001 From: Michael Gottesman Date: Sun, 15 Aug 2021 13:58:57 -0700 Subject: [PATCH] [build-script] Refactor out BuildScriptInvocation into its own file and out of the top level build-script file. This shrinks the number of lines in build-script by 50% and just makes it easier to read. It will also enable me to refactor parts of it into pieces without touching the main build-script file. This is really library code... build-script itself doesn't care about how BuildScriptInvocation is actually implemented under the hood. --- utils/build-script | 784 +---------------- .../build_script_invocation.py | 788 ++++++++++++++++++ .../swift_build_support/utils.py | 32 + 3 files changed, 826 insertions(+), 778 deletions(-) create mode 100644 utils/swift_build_support/swift_build_support/build_script_invocation.py create mode 100644 utils/swift_build_support/swift_build_support/utils.py diff --git a/utils/build-script b/utils/build-script index 360faa4ab6562..8a0be41769618 100755 --- a/utils/build-script +++ b/utils/build-script @@ -18,7 +18,6 @@ from __future__ import absolute_import, print_function, unicode_literals import json import os -import pipes import platform import sys import time @@ -35,16 +34,16 @@ from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT import six -from swift_build_support.swift_build_support import build_graph -from swift_build_support.swift_build_support import products +from swift_build_support.swift_build_support import build_script_invocation from swift_build_support.swift_build_support import shell from swift_build_support.swift_build_support import targets from swift_build_support.swift_build_support import workspace from swift_build_support.swift_build_support.cmake import CMake -from swift_build_support.swift_build_support.host_specific_configuration \ - import HostSpecificConfiguration from swift_build_support.swift_build_support.targets import StdlibDeploymentTarget from swift_build_support.swift_build_support.toolchain import host_toolchain +from swift_build_support.swift_build_support.utils \ + import exit_rejecting_arguments +from swift_build_support.swift_build_support.utils import fatal_error # ----------------------------------------------------------------------------- @@ -73,16 +72,6 @@ def print_note(message, stream=sys.stdout): stream.flush() -def fatal_error(message, stream=sys.stderr): - """Writes a message to the given stream and exits. By default this - function outputs to stderr. - """ - - stream.write('[{}] ERROR: {}\n'.format(sys.argv[0], message)) - stream.flush() - sys.exit(1) - - def clean_delay(): """Provide a short delay so accidentally invoked clean builds can be canceled. @@ -96,13 +85,6 @@ def clean_delay(): print('\b\b\b\bnow.') -def exit_rejecting_arguments(message, parser=None): - print(message, file=sys.stderr) - if parser: - parser.print_usage(sys.stderr) - sys.exit(2) # 2 is the same as `argparse` error exit code. - - def initialize_runtime_environment(): """Change the program environment for building. """ @@ -412,764 +394,10 @@ def apply_default_arguments(toolchain, args): '-DSWIFT_DARWIN_MODULE_ARCHS:STRING={}'.format( args.swift_darwin_module_archs)) - -# ----------------------------------------------------------------------------- -# Build Script Impl Wrapping - -class BuildScriptInvocation(object): - """Represent a single build script invocation. - """ - - def __init__(self, toolchain, args): - self.toolchain = toolchain - self.args = args - - self.workspace = workspace.Workspace( - source_root=SWIFT_SOURCE_ROOT, - build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir)) - - self.build_libparser_only = args.build_libparser_only - - @property - def install_all(self): - return self.args.install_all or self.args.infer_dependencies - - def build_ninja(self): - if not os.path.exists(self.workspace.source_dir("ninja")): - fatal_error( - "can't find source directory for ninja " - "(tried %s)" % (self.workspace.source_dir("ninja"))) - - ninja_build = products.Ninja.new_builder( - args=self.args, - toolchain=self.toolchain, - workspace=self.workspace, - host=StdlibDeploymentTarget.get_target_for_name( - self.args.host_target)) - ninja_build.build() - self.toolchain.ninja = ninja_build.ninja_bin_path - - def convert_to_impl_arguments(self): - """convert_to_impl_arguments() -> (env, args) - - Convert the invocation to an environment and list of arguments suitable - for invoking `build-script-impl`. - """ - - # Create local shadows, for convenience. - args = self.args - toolchain = self.toolchain - - cmake = CMake(args=args, - toolchain=self.toolchain) - - impl_args = [ - "--workspace", self.workspace.source_root, - "--build-dir", self.workspace.build_root, - "--install-prefix", args.install_prefix, - "--host-target", args.host_target, - "--stdlib-deployment-targets={}".format( - " ".join(args.stdlib_deployment_targets)), - "--host-cc", toolchain.cc, - "--host-cxx", toolchain.cxx, - "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain, - "--darwin-deployment-version-osx=%s" % ( - args.darwin_deployment_version_osx), - "--darwin-deployment-version-ios=%s" % ( - args.darwin_deployment_version_ios), - "--darwin-deployment-version-tvos=%s" % ( - args.darwin_deployment_version_tvos), - "--darwin-deployment-version-watchos=%s" % ( - args.darwin_deployment_version_watchos), - "--cmake", toolchain.cmake, - "--cmark-build-type", args.cmark_build_variant, - "--llvm-build-type", args.llvm_build_variant, - "--swift-build-type", args.swift_build_variant, - "--swift-stdlib-build-type", args.swift_stdlib_build_variant, - "--lldb-build-type", args.lldb_build_variant, - "--foundation-build-type", args.foundation_build_variant, - "--libdispatch-build-type", args.libdispatch_build_variant, - "--libicu-build-type", args.libicu_build_variant, - "--xctest-build-type", args.build_variant, - "--llbuild-build-type", args.build_variant, - "--swift-enable-assertions", str(args.swift_assertions).lower(), - "--swift-stdlib-enable-assertions", str( - args.swift_stdlib_assertions).lower(), - "--swift-analyze-code-coverage", str( - args.swift_analyze_code_coverage).lower(), - "--llbuild-enable-assertions", str( - args.llbuild_assertions).lower(), - "--lldb-assertions", str( - args.lldb_assertions).lower(), - "--cmake-generator", args.cmake_generator, - "--build-jobs", str(args.build_jobs), - "--common-cmake-options=%s" % ' '.join( - pipes.quote(opt) for opt in cmake.common_options()), - "--build-args=%s" % ' '.join( - pipes.quote(arg) for arg in cmake.build_args()), - "--dsymutil-jobs", str(args.dsymutil_jobs), - ] - - # Compute any product specific cmake arguments. - # - # NOTE: The sum(list(...)) is b/c compute_product_classes returns a - # tuple of lists of which the first is the build-script-impl products - # and the second is the non-build-script-impl-products. It guarantees - # that when we concatenate these two lists together we get a valid - # dependency graph. - for product_class in sum(list(self.compute_product_classes()), []): - if not product_class.is_build_script_impl_product(): - continue - - product_name = product_class.product_name() - product_source_name = product_class.product_source_name() - source_dir = self.workspace.source_dir(product_source_name) - - if not os.path.exists(source_dir): - fatal_error( - "can't find source directory for %s " - "(tried %s)" % (product_name, source_dir)) - - product = product_class( - args=args, - toolchain=self.toolchain, - source_dir=source_dir, - # FIXME: This is incorrect since it always assumes the host - # target I think? - build_dir=self.workspace.build_dir( - args.host_target, product_name)) - cmake_opts = product.cmake_options - - # FIXME: We should be using pipes.quote here but we run into issues - # with build-script-impl/cmake not being happy with all of the - # extra "'" in the strings. To fix this easily, we really need to - # just invoke cmake from build-script directly rather than futzing - # with build-script-impl. This makes even more sense since there - # really isn't a security issue here. - if cmake_opts: - impl_args += [ - "--{}-cmake-options={}".format( - product_name, ' '.join(cmake_opts)) - ] - - if args.build_stdlib_deployment_targets: - impl_args += [ - "--build-stdlib-deployment-targets", " ".join( - args.build_stdlib_deployment_targets)] - if args.cross_compile_hosts: - impl_args += [ - "--cross-compile-hosts", " ".join(args.cross_compile_hosts)] - - if args.test_paths: - impl_args += ["--test-paths", " ".join(args.test_paths)] - - if toolchain.ninja: - impl_args += ["--ninja-bin=%s" % toolchain.ninja] - if args.distcc: - impl_args += [ - "--distcc", - "--distcc-pump=%s" % toolchain.distcc_pump - ] - if args.sccache: - args.cmake_c_launcher = toolchain.sccache - args.cmake_cxx_launcher = toolchain.sccache - - # *NOTE* We use normal cmake to pass through tsan/ubsan options. We do - # NOT pass them to build-script-impl. - if args.enable_asan: - impl_args += ["--enable-asan"] - # If we are on linux, disable leak detection when running ASAN. We - # have a separate bot that checks for leaks. - if platform.system() == 'Linux': - os.environ['ASAN_OPTIONS'] = 'detect_leaks=0' - if args.enable_ubsan: - impl_args += ["--enable-ubsan"] - - # If we have lsan, we need to export our suppression list. The actual - # passing in of the LSAN flag is done via the normal cmake method. We - # do not pass the flag to build-script-impl. - if args.enable_lsan: - supp_file = os.path.join(SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, - "utils", - "lsan_leaks_suppression_list.txt") - os.environ['LSAN_OPTIONS'] = 'suppressions={}'.format(supp_file) - if args.verbose_build: - impl_args += ["--verbose-build"] - if args.install_symroot: - impl_args += [ - "--install-symroot", os.path.abspath(args.install_symroot) - ] - if args.install_destdir: - impl_args += [ - "--install-destdir", os.path.abspath(args.install_destdir) - ] - - if args.skip_build: - impl_args += ["--skip-build"] - if not args.build_benchmarks: - impl_args += ["--skip-build-benchmarks"] - - if args.swift_disable_dead_stripping: - args.extra_cmake_options.append('-DSWIFT_DISABLE_DEAD_STRIPPING:BOOL=TRUE') - - # Then add subproject install flags that either skip building them /or/ - # if we are going to build them and install_all is set, we also install - # them. - conditional_subproject_configs = [ - (args.build_cmark, "cmark"), - (args.build_llvm, "llvm"), - (args.build_swift, "swift"), - (args.build_foundation, "foundation"), - (args.build_xctest, "xctest"), - (args.build_lldb, "lldb"), - (args.build_llbuild, "llbuild"), - (args.build_libcxx, "libcxx"), - (args.build_libdispatch, "libdispatch"), - (args.build_libicu, "libicu") - ] - for (should_build, string_name) in conditional_subproject_configs: - if not should_build and not self.args.infer_dependencies: - impl_args += ["--skip-build-{}".format(string_name)] - elif self.install_all: - impl_args += ["--install-{}".format(string_name)] - - if args.build_swift_dynamic_stdlib: - impl_args += ["--build-swift-dynamic-stdlib"] - if args.build_swift_static_stdlib: - impl_args += ["--build-swift-static-stdlib"] - if args.build_swift_stdlib_unittest_extra: - impl_args += ["--build-swift-stdlib-unittest-extra"] - if args.build_swift_dynamic_sdk_overlay: - impl_args += ["--build-swift-dynamic-sdk-overlay"] - if args.build_swift_static_sdk_overlay: - impl_args += ["--build-swift-static-sdk-overlay"] - - if not args.build_android: - impl_args += ["--skip-build-android"] - if not args.build_clang_tools_extra: - impl_args += ["--skip-build-clang-tools-extra"] - - if not args.test and not args.long_test and not args.stress_test: - impl_args += ["--skip-test-swift"] - if not args.test: - impl_args += [ - "--skip-test-cmark", - "--skip-test-lldb", - "--skip-test-llbuild", - "--skip-test-xctest", - "--skip-test-foundation", - "--skip-test-libdispatch", - "--skip-test-libicu", - ] - if args.build_runtime_with_host_compiler: - impl_args += ["--build-runtime-with-host-compiler"] - if args.validation_test: - impl_args += ["--validation-test"] - if args.long_test: - impl_args += ["--long-test"] - if args.stress_test: - impl_args += ["--stress-test"] - if args.skip_local_build: - impl_args += ["--skip-local-build"] - if args.only_executable_test: - impl_args += ["--only-executable-test"] - if not args.benchmark: - impl_args += ["--skip-test-benchmarks"] - if args.build_libparser_only: - impl_args += ["--build-libparser-only"] - if args.android: - impl_args += [ - "--android-arch", args.android_arch, - "--android-ndk", args.android_ndk, - "--android-api-level", args.android_api_level, - "--android-ndk-gcc-version", args.android_ndk_gcc_version, - "--android-icu-uc", args.android_icu_uc, - "--android-icu-uc-include", args.android_icu_uc_include, - "--android-icu-i18n", args.android_icu_i18n, - "--android-icu-i18n-include", args.android_icu_i18n_include, - "--android-icu-data", args.android_icu_data, - ] - # If building natively on an Android host, only pass the API level. - if StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget - .host_target().name): - impl_args += ["--android-api-level", args.android_api_level] - if args.android_deploy_device_path: - impl_args += [ - "--android-deploy-device-path", - args.android_deploy_device_path, - ] - - if platform.system() == 'Darwin': - impl_args += [ - "--toolchain-prefix", - targets.darwin_toolchain_prefix( - args.install_prefix), - "--host-lipo", toolchain.lipo, - ] - - # Isolate build from the system; Darwin toolchains build against SDKs. - # For additional isolation, disable pkg-config. Homebrew's pkg-config - # prioritizes CommandLineTools paths, resulting in compile errors. - args.extra_cmake_options += [ - '-DCMAKE_IGNORE_PATH=/usr/lib;/usr/local/lib;/lib', - '-DPKG_CONFIG_EXECUTABLE=/usr/bin/false', - ] - - if toolchain.libtool is not None: - impl_args += [ - "--host-libtool", toolchain.libtool, - ] - if args.native_clang_tools_path is not None: - impl_args += [ - "--native-clang-tools-path=%s" % args.native_clang_tools_path - ] - if args.native_llvm_tools_path is not None: - impl_args += [ - "--native-llvm-tools-path=%s" % args.native_llvm_tools_path - ] - if args.native_swift_tools_path is not None: - impl_args += [ - "--native-swift-tools-path=%s" % args.native_swift_tools_path - ] - - # If we have extra_swift_args, combine all of them together and then - # add them as one command. - if args.extra_swift_args: - impl_args += [ - "--extra-swift-args=%s" % ';'.join(args.extra_swift_args) - ] - - # Enable macCatalyst - if args.maccatalyst: - (args.extra_cmake_options - .append('-DSWIFT_ENABLE_MACCATALYST:BOOL=TRUE')) - if args.maccatalyst_ios_tests: - impl_args += ["--darwin-test-maccatalyst-ios-like=1"] - - # If we have extra_cmake_options, combine all of them together and then - # add them as one command. - if args.extra_cmake_options: - impl_args += [ - "--extra-cmake-options=%s" % ' '.join( - pipes.quote(opt) for opt in args.extra_cmake_options) - ] - - if args.lto_type is not None: - impl_args += [ - "--llvm-enable-lto=%s" % args.lto_type, - "--swift-tools-enable-lto=%s" % args.lto_type - ] - if args.llvm_max_parallel_lto_link_jobs is not None: - impl_args += [ - "--llvm-num-parallel-lto-link-jobs=%s" % - min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs) - ] - if args.swift_tools_max_parallel_lto_link_jobs is not None: - impl_args += [ - "--swift-tools-num-parallel-lto-link-jobs=%s" % - min(args.swift_tools_max_parallel_lto_link_jobs, - args.build_jobs) - ] - - impl_args += args.build_script_impl_args - - if args.dry_run: - impl_args += ["--dry-run"] - - if args.reconfigure: - impl_args += ["--reconfigure"] - - if args.clang_profile_instr_use: - impl_args += [ - "--clang-profile-instr-use=%s" % - os.path.abspath(args.clang_profile_instr_use) - ] - - if args.lit_args: - impl_args += ["--llvm-lit-args=%s" % args.lit_args] - - if args.coverage_db: - impl_args += [ - "--coverage-db=%s" % - os.path.abspath(args.coverage_db) - ] - - if args.llvm_install_components: - impl_args += [ - "--llvm-install-components=%s" % args.llvm_install_components - ] - - if not args.clean_llbuild: - impl_args += [ - "--skip-clean-llbuild" - ] - - if args.llvm_ninja_targets: - impl_args += [ - "--llvm-ninja-targets=%s" % ' '.join(args.llvm_ninja_targets) - ] - - if args.llvm_ninja_targets_for_cross_compile_hosts: - impl_args += [ - "--llvm-ninja-targets-for-cross-compile-hosts=%s" % - ' '.join(args.llvm_ninja_targets_for_cross_compile_hosts) - ] - - if args.darwin_symroot_path_filters: - impl_args += [ - "--darwin_symroot_path_filters=%s" % - ' '.join(args.darwin_symroot_path_filters) - ] - - # Compute the set of host-specific variables, which we pass through to - # the build script via environment variables. - host_specific_variables = self.compute_host_specific_variables() - impl_env = {} - for (host_target, options) in host_specific_variables.items(): - for (name, value) in options.items(): - # We mangle into an environment variable we can easily evaluate - # from the `build-script-impl`. - impl_env["HOST_VARIABLE_{}__{}".format( - host_target.replace("-", "_"), name)] = value - - return (impl_env, impl_args) - - def compute_host_specific_variables(self): - """compute_host_specific_variables(args) -> dict - - Compute the host-specific options, organized as a dictionary keyed by - host of options. - """ - - args = self.args - args.build_root = self.workspace.build_root - - options = {} - for host_target in [args.host_target] + args.cross_compile_hosts: - # Compute the host specific configuration. - try: - config = HostSpecificConfiguration(host_target, args) - except argparse.ArgumentError as e: - exit_rejecting_arguments(six.text_type(e)) - - # Convert into `build-script-impl` style variables. - options[host_target] = { - "SWIFT_SDKS": " ".join(sorted( - config.sdks_to_configure)), - "SWIFT_STDLIB_TARGETS": " ".join( - config.swift_stdlib_build_targets), - "SWIFT_BENCHMARK_TARGETS": " ".join( - config.swift_benchmark_build_targets), - "SWIFT_RUN_BENCHMARK_TARGETS": " ".join( - config.swift_benchmark_run_targets), - "SWIFT_TEST_TARGETS": " ".join( - config.swift_test_run_targets), - "SWIFT_FLAGS": config.swift_flags, - "SWIFT_TARGET_CMAKE_OPTIONS": config.cmake_options, - } - - return options - - def compute_product_classes(self): - """compute_product_classes() -> (list, list, list) - - Compute the list first of all pre-build-script-impl products, then all - build-script-impl products and then all non-build-script-impl products. - It is assumed that concatenating the three lists together will result in a - valid dependency graph for the compilation. - """ - before_impl_product_classes = [] - # If --skip-early-swift-driver is passed in, swift will be built - # as usual, but relying on its own C++-based (Legacy) driver. - # Otherwise, we build an "early" swift-driver using the host - # toolchain, which the later-built compiler will forward - # `swiftc` invocations to. That is, if we find a Swift compiler - # in the host toolchain. If the host toolchain is not equpipped with - # a Swift compiler, a warning is emitted. In the future, it may become - # mandatory that the host toolchain come with its own `swiftc`. - if self.args.build_early_swift_driver: - before_impl_product_classes.append(products.EarlySwiftDriver) - - if self.args.build_cmark: - before_impl_product_classes.append(products.CMark) - - # FIXME: This is a weird division (returning a list of class objects), - # but it matches the existing structure of the `build-script-impl`. - impl_product_classes = [] - - # If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as - # Swift still needs a few LLVM targets like tblgen to be built for it to be - # configured. Instead, handle this in build-script-impl for now. - impl_product_classes.append(products.LLVM) - if self.args.build_libcxx: - impl_product_classes.append(products.LibCXX) - if self.args.build_libicu: - impl_product_classes.append(products.LibICU) - if self.args.build_swift: - impl_product_classes.append(products.Swift) - if self.args.build_lldb: - impl_product_classes.append(products.LLDB) - if self.args.build_libdispatch: - impl_product_classes.append(products.LibDispatch) - if self.args.build_foundation: - impl_product_classes.append(products.Foundation) - if self.args.build_xctest: - impl_product_classes.append(products.XCTest) - if self.args.build_llbuild: - impl_product_classes.append(products.LLBuild) - # Sanity check that all of our impl classes are actually - # build_script_impl products. - for prod in impl_product_classes: - assert(prod.is_build_script_impl_product()) - - product_classes = [] - if self.args.build_swiftpm: - product_classes.append(products.SwiftPM) - if self.args.build_swiftsyntax: - product_classes.append(products.SwiftSyntax) - if self.args.build_skstresstester: - product_classes.append(products.SKStressTester) - if self.args.build_swiftformat: - product_classes.append(products.SwiftFormat) - if self.args.build_swiftevolve: - product_classes.append(products.SwiftEvolve) - if self.args.build_indexstoredb: - product_classes.append(products.IndexStoreDB) - if self.args.build_playgroundsupport: - product_classes.append(products.PlaygroundSupport) - if self.args.build_sourcekitlsp: - product_classes.append(products.SourceKitLSP) - if self.args.build_toolchainbenchmarks: - product_classes.append(products.Benchmarks) - if self.args.build_swift_inspect: - product_classes.append(products.SwiftInspect) - if self.args.tsan_libdispatch_test: - product_classes.append(products.TSanLibDispatch) - - # Keep SwiftDriver at last. - # swift-driver's integration with the build scripts is not fully - # supported. Using swift-driver to build these products may hit - # failures. - if self.args.build_swift_driver or self.args.install_swift_driver: - product_classes.append(products.SwiftDriver) - # Sanity check that all of our non-impl classes are actually - # not build_script_impl products. - for prod in product_classes: - assert(not prod.is_build_script_impl_product()) - - # Now that we have our two lists of product_classes, if we are asked to - # infer dependencies, infer the dependencies now and then re-split the - # list. - if self.args.infer_dependencies: - combined_classes = before_impl_product_classes +\ - impl_product_classes +\ - product_classes - if self.args.verbose_build: - print("-- Build Graph Inference --") - print("Initial Product List:") - for p in combined_classes: - print(" {}".format(p.product_name())) - - # Now that we have produced the schedule, resplit. We require our - # dependencies to respect our build-script-impl property. This means - # that no build-script-impl products can have dependencies on - # non-build-script-impl products. Otherwise, it would be unsafe to - # re-order build-script-impl products in front of non - # build-script-impl products. - before_impl_product_classes = [] - impl_product_classes = [] - product_classes = [] - is_darwin = platform.system() == 'Darwin' - final_schedule =\ - build_graph.produce_scheduled_build(combined_classes)[0] - for p in final_schedule: - if is_darwin and p.is_nondarwin_only_build_product(): - continue - if p.is_build_script_impl_product(): - impl_product_classes.append(p) - elif p.is_before_build_script_impl_product(): - before_impl_product_classes.append(p) - else: - product_classes.append(p) - - if self.args.verbose_build: - print("Final Build Order:") - for p in before_impl_product_classes: - print(" {}".format(p.product_name())) - for p in impl_product_classes: - print(" {}".format(p.product_name())) - for p in product_classes: - print(" {}".format(p.product_name())) - return (before_impl_product_classes, impl_product_classes, product_classes) - - def execute(self): - """Execute the invocation with the configured arguments.""" - - # Convert to a build-script-impl invocation. - (self.impl_env, self.impl_args) = self.convert_to_impl_arguments() - - # If using the legacy implementation, delegate all behavior to - # `build-script-impl`. - if self.args.legacy_impl: - # Execute the underlying build script implementation. - shell.call_without_sleeping([BUILD_SCRIPT_IMPL_PATH] + self.impl_args, - env=self.impl_env, echo=True) - return - - # Otherwise, we compute and execute the individual actions ourselves. - # Compute the list of hosts to operate on. - all_host_names = [ - self.args.host_target] + self.args.cross_compile_hosts - all_hosts = [StdlibDeploymentTarget.get_target_for_name(name) - for name in all_host_names] - - # Compute the list of product classes to operate on. - # - # FIXME: This should really be per-host, but the current structure - # matches that of `build-script-impl`. - (before_impl_product_classes, impl_product_classes, product_classes) =\ - self.compute_product_classes() - - # Execute each "pass". - - # Pre-build-script-impl products... - # Note: currently only supports building for the host. - for host_target in all_host_names: - for product_class in before_impl_product_classes: - if product_class.is_build_script_impl_product(): - continue - if not product_class.is_before_build_script_impl_product(): - continue - # Execute clean, build, test, install - self.execute_product_build_steps(product_class, host_target) - - # Build... - for host_target in all_hosts: - # FIXME: We should only compute these once. - try: - config = HostSpecificConfiguration(host_target.name, self.args) - except argparse.ArgumentError as e: - exit_rejecting_arguments(six.text_type(e)) - print("Building the standard library for: {}".format( - " ".join(config.swift_stdlib_build_targets))) - if config.swift_test_run_targets and ( - self.args.test or self.args.long_test): - print("Running Swift tests for: {}".format( - " ".join(config.swift_test_run_targets))) - if config.swift_benchmark_run_targets and self.args.benchmark: - print("Running Swift benchmarks for: {}".format( - " ".join(config.swift_benchmark_run_targets))) - - for product_class in impl_product_classes: - self._execute_build_action(host_target, product_class) - - # Test... - for host_target in all_hosts: - for product_class in impl_product_classes: - self._execute_test_action(host_target, product_class) - - # Install... - for host_target in all_hosts: - for product_class in impl_product_classes: - self._execute_install_action(host_target, product_class) - - # Core Lipo... - self._execute_merged_host_lipo_core_action() - - # Non-build-script-impl products... - # Note: currently only supports building for the host. - for host_target in [self.args.host_target]: - for product_class in product_classes: - if product_class.is_build_script_impl_product(): - continue - # Execute clean, build, test, install - self.execute_product_build_steps(product_class, host_target) - - # Extract symbols... - for host_target in all_hosts: - self._execute_extract_symbols_action(host_target) - - # Package... - for host_target in all_hosts: - self._execute_package_action(host_target) - - # Lipo... - self._execute_merged_host_lipo_action() - - def _execute_build_action(self, host_target, product_class): - action_name = "{}-{}-build".format(host_target.name, - product_class.product_name()) - self._execute_action(action_name) - - def _execute_test_action(self, host_target, product_class): - action_name = "{}-{}-test".format(host_target.name, - product_class.product_name()) - self._execute_action(action_name) - - def _execute_install_action(self, host_target, product_class): - action_name = "{}-{}-install".format(host_target.name, - product_class.product_name()) - self._execute_action(action_name) - - def _execute_extract_symbols_action(self, host_target): - action_name = "{}-extractsymbols".format(host_target.name) - self._execute_action(action_name) - - def _execute_package_action(self, host_target): - action_name = "{}-package".format(host_target.name) - self._execute_action(action_name) - - def _execute_merged_host_lipo_action(self): - self._execute_action("merged-hosts-lipo") - - def _execute_merged_host_lipo_core_action(self): - self._execute_action("merged-hosts-lipo-core") - - def _execute_action(self, action_name): - shell.call_without_sleeping( - [BUILD_SCRIPT_IMPL_PATH] + self.impl_args + - ["--only-execute", action_name], - env=self.impl_env, echo=self.args.verbose_build) - - def execute_product_build_steps(self, product_class, host_target): - product_source = product_class.product_source_name() - product_name = product_class.product_name() - if product_class.is_swiftpm_unified_build_product(): - build_dir = self.workspace.swiftpm_unified_build_dir( - host_target) - else: - build_dir = self.workspace.build_dir( - host_target, product_name) - product = product_class( - args=self.args, - toolchain=self.toolchain, - source_dir=self.workspace.source_dir(product_source), - build_dir=build_dir) - if product.should_clean(host_target): - print("--- Cleaning %s ---" % product_name) - product.clean(host_target) - if product.should_build(host_target): - print("--- Building %s ---" % product_name) - product.build(host_target) - if product.should_test(host_target): - print("--- Running tests for %s ---" % product_name) - product.test(host_target) - print("--- Finished tests for %s ---" % product_name) - # Install the product if it should be installed specifically, or - # if it should be built and `install_all` is set to True. - # The exception is select before_build_script_impl products - # which set `is_ignore_install_all_product` to True, ensuring - # they are never installed. (e.g. earlySwiftDriver). - if product.should_install(host_target) or \ - (self.install_all and product.should_build(host_target) and - not product.is_ignore_install_all_product()): - print("--- Installing %s ---" % product_name) - product.install(host_target) - - # ----------------------------------------------------------------------------- # Main (preset) + def parse_preset_args(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, @@ -1409,7 +637,7 @@ def main_normal(): validate_arguments(toolchain, args) # Create the build script invocation. - invocation = BuildScriptInvocation(toolchain, args) + invocation = build_script_invocation.BuildScriptInvocation(toolchain, args) # Sanitize the runtime environment. initialize_runtime_environment() diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py new file mode 100644 index 0000000000000..0e98b953df8e0 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -0,0 +1,788 @@ +# ===-- build_script_invocation.py ---------------------------------------===# +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https:#swift.org/LICENSE.txt for license information +# See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ===---------------------------------------------------------------------===# + +import os +import pipes +import platform + +from build_swift.build_swift import argparse +from build_swift.build_swift.constants import BUILD_SCRIPT_IMPL_PATH +from build_swift.build_swift.constants import SWIFT_BUILD_ROOT +from build_swift.build_swift.constants import SWIFT_REPO_NAME +from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT + +import six + +from swift_build_support.swift_build_support import build_graph +from swift_build_support.swift_build_support import products +from swift_build_support.swift_build_support import shell +from swift_build_support.swift_build_support import targets +from swift_build_support.swift_build_support import workspace +from swift_build_support.swift_build_support.cmake import CMake +from swift_build_support.swift_build_support.host_specific_configuration \ + import HostSpecificConfiguration +from swift_build_support.swift_build_support.targets \ + import StdlibDeploymentTarget +from swift_build_support.swift_build_support.utils \ + import exit_rejecting_arguments +from swift_build_support.swift_build_support.utils import fatal_error + + +class BuildScriptInvocation(object): + """Represent a single build script invocation. + """ + + def __init__(self, toolchain, args): + self.toolchain = toolchain + self.args = args + + self.workspace = workspace.Workspace( + source_root=SWIFT_SOURCE_ROOT, + build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir)) + + self.build_libparser_only = args.build_libparser_only + + @property + def install_all(self): + return self.args.install_all or self.args.infer_dependencies + + def build_ninja(self): + if not os.path.exists(self.workspace.source_dir("ninja")): + fatal_error( + "can't find source directory for ninja " + "(tried %s)" % (self.workspace.source_dir("ninja"))) + + ninja_build = products.Ninja.new_builder( + args=self.args, + toolchain=self.toolchain, + workspace=self.workspace, + host=StdlibDeploymentTarget.get_target_for_name( + self.args.host_target)) + ninja_build.build() + self.toolchain.ninja = ninja_build.ninja_bin_path + + def convert_to_impl_arguments(self): + """convert_to_impl_arguments() -> (env, args) + + Convert the invocation to an environment and list of arguments suitable + for invoking `build-script-impl`. + """ + + # Create local shadows, for convenience. + args = self.args + toolchain = self.toolchain + + cmake = CMake(args=args, + toolchain=self.toolchain) + + impl_args = [ + "--workspace", self.workspace.source_root, + "--build-dir", self.workspace.build_root, + "--install-prefix", args.install_prefix, + "--host-target", args.host_target, + "--stdlib-deployment-targets={}".format( + " ".join(args.stdlib_deployment_targets)), + "--host-cc", toolchain.cc, + "--host-cxx", toolchain.cxx, + "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain, + "--darwin-deployment-version-osx=%s" % ( + args.darwin_deployment_version_osx), + "--darwin-deployment-version-ios=%s" % ( + args.darwin_deployment_version_ios), + "--darwin-deployment-version-tvos=%s" % ( + args.darwin_deployment_version_tvos), + "--darwin-deployment-version-watchos=%s" % ( + args.darwin_deployment_version_watchos), + "--cmake", toolchain.cmake, + "--cmark-build-type", args.cmark_build_variant, + "--llvm-build-type", args.llvm_build_variant, + "--swift-build-type", args.swift_build_variant, + "--swift-stdlib-build-type", args.swift_stdlib_build_variant, + "--lldb-build-type", args.lldb_build_variant, + "--foundation-build-type", args.foundation_build_variant, + "--libdispatch-build-type", args.libdispatch_build_variant, + "--libicu-build-type", args.libicu_build_variant, + "--xctest-build-type", args.build_variant, + "--llbuild-build-type", args.build_variant, + "--swift-enable-assertions", str(args.swift_assertions).lower(), + "--swift-stdlib-enable-assertions", str( + args.swift_stdlib_assertions).lower(), + "--swift-analyze-code-coverage", str( + args.swift_analyze_code_coverage).lower(), + "--llbuild-enable-assertions", str( + args.llbuild_assertions).lower(), + "--lldb-assertions", str( + args.lldb_assertions).lower(), + "--cmake-generator", args.cmake_generator, + "--build-jobs", str(args.build_jobs), + "--common-cmake-options=%s" % ' '.join( + pipes.quote(opt) for opt in cmake.common_options()), + "--build-args=%s" % ' '.join( + pipes.quote(arg) for arg in cmake.build_args()), + "--dsymutil-jobs", str(args.dsymutil_jobs), + ] + + # Compute any product specific cmake arguments. + # + # NOTE: The sum(list(...)) is b/c compute_product_classes returns a + # tuple of lists of which the first is the build-script-impl products + # and the second is the non-build-script-impl-products. It guarantees + # that when we concatenate these two lists together we get a valid + # dependency graph. + for product_class in sum(list(self.compute_product_classes()), []): + if not product_class.is_build_script_impl_product(): + continue + + product_name = product_class.product_name() + product_source_name = product_class.product_source_name() + source_dir = self.workspace.source_dir(product_source_name) + + if not os.path.exists(source_dir): + fatal_error( + "can't find source directory for %s " + "(tried %s)" % (product_name, source_dir)) + + product = product_class( + args=args, + toolchain=self.toolchain, + source_dir=source_dir, + # FIXME: This is incorrect since it always assumes the host + # target I think? + build_dir=self.workspace.build_dir( + args.host_target, product_name)) + cmake_opts = product.cmake_options + + # FIXME: We should be using pipes.quote here but we run into issues + # with build-script-impl/cmake not being happy with all of the + # extra "'" in the strings. To fix this easily, we really need to + # just invoke cmake from build-script directly rather than futzing + # with build-script-impl. This makes even more sense since there + # really isn't a security issue here. + if cmake_opts: + impl_args += [ + "--{}-cmake-options={}".format( + product_name, ' '.join(cmake_opts)) + ] + + if args.build_stdlib_deployment_targets: + impl_args += [ + "--build-stdlib-deployment-targets", " ".join( + args.build_stdlib_deployment_targets)] + if args.cross_compile_hosts: + impl_args += [ + "--cross-compile-hosts", " ".join(args.cross_compile_hosts)] + + if args.test_paths: + impl_args += ["--test-paths", " ".join(args.test_paths)] + + if toolchain.ninja: + impl_args += ["--ninja-bin=%s" % toolchain.ninja] + if args.distcc: + impl_args += [ + "--distcc", + "--distcc-pump=%s" % toolchain.distcc_pump + ] + if args.sccache: + args.cmake_c_launcher = toolchain.sccache + args.cmake_cxx_launcher = toolchain.sccache + + # *NOTE* We use normal cmake to pass through tsan/ubsan options. We do + # NOT pass them to build-script-impl. + if args.enable_asan: + impl_args += ["--enable-asan"] + # If we are on linux, disable leak detection when running ASAN. We + # have a separate bot that checks for leaks. + if platform.system() == 'Linux': + os.environ['ASAN_OPTIONS'] = 'detect_leaks=0' + if args.enable_ubsan: + impl_args += ["--enable-ubsan"] + + # If we have lsan, we need to export our suppression list. The actual + # passing in of the LSAN flag is done via the normal cmake method. We + # do not pass the flag to build-script-impl. + if args.enable_lsan: + supp_file = os.path.join(SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, + "utils", + "lsan_leaks_suppression_list.txt") + os.environ['LSAN_OPTIONS'] = 'suppressions={}'.format(supp_file) + if args.verbose_build: + impl_args += ["--verbose-build"] + if args.install_symroot: + impl_args += [ + "--install-symroot", os.path.abspath(args.install_symroot) + ] + if args.install_destdir: + impl_args += [ + "--install-destdir", os.path.abspath(args.install_destdir) + ] + + if args.skip_build: + impl_args += ["--skip-build"] + if not args.build_benchmarks: + impl_args += ["--skip-build-benchmarks"] + + if args.swift_disable_dead_stripping: + args.extra_cmake_options.append('-DSWIFT_DISABLE_DEAD_STRIPPING:BOOL=TRUE') + + # Then add subproject install flags that either skip building them /or/ + # if we are going to build them and install_all is set, we also install + # them. + conditional_subproject_configs = [ + (args.build_cmark, "cmark"), + (args.build_llvm, "llvm"), + (args.build_swift, "swift"), + (args.build_foundation, "foundation"), + (args.build_xctest, "xctest"), + (args.build_lldb, "lldb"), + (args.build_llbuild, "llbuild"), + (args.build_libcxx, "libcxx"), + (args.build_libdispatch, "libdispatch"), + (args.build_libicu, "libicu") + ] + for (should_build, string_name) in conditional_subproject_configs: + if not should_build and not self.args.infer_dependencies: + impl_args += ["--skip-build-{}".format(string_name)] + elif self.install_all: + impl_args += ["--install-{}".format(string_name)] + + if args.build_swift_dynamic_stdlib: + impl_args += ["--build-swift-dynamic-stdlib"] + if args.build_swift_static_stdlib: + impl_args += ["--build-swift-static-stdlib"] + if args.build_swift_stdlib_unittest_extra: + impl_args += ["--build-swift-stdlib-unittest-extra"] + if args.build_swift_dynamic_sdk_overlay: + impl_args += ["--build-swift-dynamic-sdk-overlay"] + if args.build_swift_static_sdk_overlay: + impl_args += ["--build-swift-static-sdk-overlay"] + + if not args.build_android: + impl_args += ["--skip-build-android"] + if not args.build_clang_tools_extra: + impl_args += ["--skip-build-clang-tools-extra"] + + if not args.test and not args.long_test and not args.stress_test: + impl_args += ["--skip-test-swift"] + if not args.test: + impl_args += [ + "--skip-test-cmark", + "--skip-test-lldb", + "--skip-test-llbuild", + "--skip-test-xctest", + "--skip-test-foundation", + "--skip-test-libdispatch", + "--skip-test-libicu", + ] + if args.build_runtime_with_host_compiler: + impl_args += ["--build-runtime-with-host-compiler"] + if args.validation_test: + impl_args += ["--validation-test"] + if args.long_test: + impl_args += ["--long-test"] + if args.stress_test: + impl_args += ["--stress-test"] + if args.skip_local_build: + impl_args += ["--skip-local-build"] + if args.only_executable_test: + impl_args += ["--only-executable-test"] + if not args.benchmark: + impl_args += ["--skip-test-benchmarks"] + if args.build_libparser_only: + impl_args += ["--build-libparser-only"] + if args.android: + impl_args += [ + "--android-arch", args.android_arch, + "--android-ndk", args.android_ndk, + "--android-api-level", args.android_api_level, + "--android-ndk-gcc-version", args.android_ndk_gcc_version, + "--android-icu-uc", args.android_icu_uc, + "--android-icu-uc-include", args.android_icu_uc_include, + "--android-icu-i18n", args.android_icu_i18n, + "--android-icu-i18n-include", args.android_icu_i18n_include, + "--android-icu-data", args.android_icu_data, + ] + # If building natively on an Android host, only pass the API level. + if StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget + .host_target().name): + impl_args += ["--android-api-level", args.android_api_level] + if args.android_deploy_device_path: + impl_args += [ + "--android-deploy-device-path", + args.android_deploy_device_path, + ] + + if platform.system() == 'Darwin': + impl_args += [ + "--toolchain-prefix", + targets.darwin_toolchain_prefix( + args.install_prefix), + "--host-lipo", toolchain.lipo, + ] + + # Isolate build from the system; Darwin toolchains build against SDKs. + # For additional isolation, disable pkg-config. Homebrew's pkg-config + # prioritizes CommandLineTools paths, resulting in compile errors. + args.extra_cmake_options += [ + '-DCMAKE_IGNORE_PATH=/usr/lib;/usr/local/lib;/lib', + '-DPKG_CONFIG_EXECUTABLE=/usr/bin/false', + ] + + if toolchain.libtool is not None: + impl_args += [ + "--host-libtool", toolchain.libtool, + ] + if args.native_clang_tools_path is not None: + impl_args += [ + "--native-clang-tools-path=%s" % args.native_clang_tools_path + ] + if args.native_llvm_tools_path is not None: + impl_args += [ + "--native-llvm-tools-path=%s" % args.native_llvm_tools_path + ] + if args.native_swift_tools_path is not None: + impl_args += [ + "--native-swift-tools-path=%s" % args.native_swift_tools_path + ] + + # If we have extra_swift_args, combine all of them together and then + # add them as one command. + if args.extra_swift_args: + impl_args += [ + "--extra-swift-args=%s" % ';'.join(args.extra_swift_args) + ] + + # Enable macCatalyst + if args.maccatalyst: + (args.extra_cmake_options + .append('-DSWIFT_ENABLE_MACCATALYST:BOOL=TRUE')) + if args.maccatalyst_ios_tests: + impl_args += ["--darwin-test-maccatalyst-ios-like=1"] + + # If we have extra_cmake_options, combine all of them together and then + # add them as one command. + if args.extra_cmake_options: + impl_args += [ + "--extra-cmake-options=%s" % ' '.join( + pipes.quote(opt) for opt in args.extra_cmake_options) + ] + + if args.lto_type is not None: + impl_args += [ + "--llvm-enable-lto=%s" % args.lto_type, + "--swift-tools-enable-lto=%s" % args.lto_type + ] + if args.llvm_max_parallel_lto_link_jobs is not None: + impl_args += [ + "--llvm-num-parallel-lto-link-jobs=%s" % + min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs) + ] + if args.swift_tools_max_parallel_lto_link_jobs is not None: + impl_args += [ + "--swift-tools-num-parallel-lto-link-jobs=%s" % + min(args.swift_tools_max_parallel_lto_link_jobs, + args.build_jobs) + ] + + impl_args += args.build_script_impl_args + + if args.dry_run: + impl_args += ["--dry-run"] + + if args.reconfigure: + impl_args += ["--reconfigure"] + + if args.clang_profile_instr_use: + impl_args += [ + "--clang-profile-instr-use=%s" % + os.path.abspath(args.clang_profile_instr_use) + ] + + if args.lit_args: + impl_args += ["--llvm-lit-args=%s" % args.lit_args] + + if args.coverage_db: + impl_args += [ + "--coverage-db=%s" % + os.path.abspath(args.coverage_db) + ] + + if args.llvm_install_components: + impl_args += [ + "--llvm-install-components=%s" % args.llvm_install_components + ] + + if not args.clean_llbuild: + impl_args += [ + "--skip-clean-llbuild" + ] + + if args.llvm_ninja_targets: + impl_args += [ + "--llvm-ninja-targets=%s" % ' '.join(args.llvm_ninja_targets) + ] + + if args.llvm_ninja_targets_for_cross_compile_hosts: + impl_args += [ + "--llvm-ninja-targets-for-cross-compile-hosts=%s" % + ' '.join(args.llvm_ninja_targets_for_cross_compile_hosts) + ] + + if args.darwin_symroot_path_filters: + impl_args += [ + "--darwin_symroot_path_filters=%s" % + ' '.join(args.darwin_symroot_path_filters) + ] + + # Compute the set of host-specific variables, which we pass through to + # the build script via environment variables. + host_specific_variables = self.compute_host_specific_variables() + impl_env = {} + for (host_target, options) in host_specific_variables.items(): + for (name, value) in options.items(): + # We mangle into an environment variable we can easily evaluate + # from the `build-script-impl`. + impl_env["HOST_VARIABLE_{}__{}".format( + host_target.replace("-", "_"), name)] = value + + return (impl_env, impl_args) + + def compute_host_specific_variables(self): + """compute_host_specific_variables(args) -> dict + + Compute the host-specific options, organized as a dictionary keyed by + host of options. + """ + + args = self.args + args.build_root = self.workspace.build_root + + options = {} + for host_target in [args.host_target] + args.cross_compile_hosts: + # Compute the host specific configuration. + try: + config = HostSpecificConfiguration(host_target, args) + except argparse.ArgumentError as e: + exit_rejecting_arguments(six.text_type(e)) + + # Convert into `build-script-impl` style variables. + options[host_target] = { + "SWIFT_SDKS": " ".join(sorted( + config.sdks_to_configure)), + "SWIFT_STDLIB_TARGETS": " ".join( + config.swift_stdlib_build_targets), + "SWIFT_BENCHMARK_TARGETS": " ".join( + config.swift_benchmark_build_targets), + "SWIFT_RUN_BENCHMARK_TARGETS": " ".join( + config.swift_benchmark_run_targets), + "SWIFT_TEST_TARGETS": " ".join( + config.swift_test_run_targets), + "SWIFT_FLAGS": config.swift_flags, + "SWIFT_TARGET_CMAKE_OPTIONS": config.cmake_options, + } + + return options + + def compute_product_classes(self): + """compute_product_classes() -> (list, list, list) + + Compute the list first of all pre-build-script-impl products, then all + build-script-impl products and then all non-build-script-impl products. + It is assumed that concatenating the three lists together will result in a + valid dependency graph for the compilation. + """ + before_impl_product_classes = [] + # If --skip-early-swift-driver is passed in, swift will be built + # as usual, but relying on its own C++-based (Legacy) driver. + # Otherwise, we build an "early" swift-driver using the host + # toolchain, which the later-built compiler will forward + # `swiftc` invocations to. That is, if we find a Swift compiler + # in the host toolchain. If the host toolchain is not equpipped with + # a Swift compiler, a warning is emitted. In the future, it may become + # mandatory that the host toolchain come with its own `swiftc`. + if self.args.build_early_swift_driver: + before_impl_product_classes.append(products.EarlySwiftDriver) + + if self.args.build_cmark: + before_impl_product_classes.append(products.CMark) + + # FIXME: This is a weird division (returning a list of class objects), + # but it matches the existing structure of the `build-script-impl`. + impl_product_classes = [] + + # If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as + # Swift still needs a few LLVM targets like tblgen to be built for it to be + # configured. Instead, handle this in build-script-impl for now. + impl_product_classes.append(products.LLVM) + if self.args.build_libcxx: + impl_product_classes.append(products.LibCXX) + if self.args.build_libicu: + impl_product_classes.append(products.LibICU) + if self.args.build_swift: + impl_product_classes.append(products.Swift) + if self.args.build_lldb: + impl_product_classes.append(products.LLDB) + if self.args.build_libdispatch: + impl_product_classes.append(products.LibDispatch) + if self.args.build_foundation: + impl_product_classes.append(products.Foundation) + if self.args.build_xctest: + impl_product_classes.append(products.XCTest) + if self.args.build_llbuild: + impl_product_classes.append(products.LLBuild) + # Sanity check that all of our impl classes are actually + # build_script_impl products. + for prod in impl_product_classes: + assert(prod.is_build_script_impl_product()) + + product_classes = [] + if self.args.build_swiftpm: + product_classes.append(products.SwiftPM) + if self.args.build_swiftsyntax: + product_classes.append(products.SwiftSyntax) + if self.args.build_skstresstester: + product_classes.append(products.SKStressTester) + if self.args.build_swiftformat: + product_classes.append(products.SwiftFormat) + if self.args.build_swiftevolve: + product_classes.append(products.SwiftEvolve) + if self.args.build_indexstoredb: + product_classes.append(products.IndexStoreDB) + if self.args.build_playgroundsupport: + product_classes.append(products.PlaygroundSupport) + if self.args.build_sourcekitlsp: + product_classes.append(products.SourceKitLSP) + if self.args.build_toolchainbenchmarks: + product_classes.append(products.Benchmarks) + if self.args.build_swift_inspect: + product_classes.append(products.SwiftInspect) + if self.args.tsan_libdispatch_test: + product_classes.append(products.TSanLibDispatch) + + # Keep SwiftDriver at last. + # swift-driver's integration with the build scripts is not fully + # supported. Using swift-driver to build these products may hit + # failures. + if self.args.build_swift_driver or self.args.install_swift_driver: + product_classes.append(products.SwiftDriver) + # Sanity check that all of our non-impl classes are actually + # not build_script_impl products. + for prod in product_classes: + assert(not prod.is_build_script_impl_product()) + + # Now that we have our two lists of product_classes, if we are asked to + # infer dependencies, infer the dependencies now and then re-split the + # list. + if self.args.infer_dependencies: + combined_classes = before_impl_product_classes +\ + impl_product_classes +\ + product_classes + if self.args.verbose_build: + print("-- Build Graph Inference --") + print("Initial Product List:") + for p in combined_classes: + print(" {}".format(p.product_name())) + + # Now that we have produced the schedule, resplit. We require our + # dependencies to respect our build-script-impl property. This means + # that no build-script-impl products can have dependencies on + # non-build-script-impl products. Otherwise, it would be unsafe to + # re-order build-script-impl products in front of non + # build-script-impl products. + before_impl_product_classes = [] + impl_product_classes = [] + product_classes = [] + is_darwin = platform.system() == 'Darwin' + final_schedule =\ + build_graph.produce_scheduled_build(combined_classes)[0] + for p in final_schedule: + if is_darwin and p.is_nondarwin_only_build_product(): + continue + if p.is_build_script_impl_product(): + impl_product_classes.append(p) + elif p.is_before_build_script_impl_product(): + before_impl_product_classes.append(p) + else: + product_classes.append(p) + + if self.args.verbose_build: + print("Final Build Order:") + for p in before_impl_product_classes: + print(" {}".format(p.product_name())) + for p in impl_product_classes: + print(" {}".format(p.product_name())) + for p in product_classes: + print(" {}".format(p.product_name())) + return (before_impl_product_classes, impl_product_classes, product_classes) + + def execute(self): + """Execute the invocation with the configured arguments.""" + + # Convert to a build-script-impl invocation. + (self.impl_env, self.impl_args) = self.convert_to_impl_arguments() + + # If using the legacy implementation, delegate all behavior to + # `build-script-impl`. + if self.args.legacy_impl: + # Execute the underlying build script implementation. + shell.call_without_sleeping([BUILD_SCRIPT_IMPL_PATH] + self.impl_args, + env=self.impl_env, echo=True) + return + + # Otherwise, we compute and execute the individual actions ourselves. + # Compute the list of hosts to operate on. + all_host_names = [ + self.args.host_target] + self.args.cross_compile_hosts + all_hosts = [StdlibDeploymentTarget.get_target_for_name(name) + for name in all_host_names] + + # Compute the list of product classes to operate on. + # + # FIXME: This should really be per-host, but the current structure + # matches that of `build-script-impl`. + (before_impl_product_classes, impl_product_classes, product_classes) =\ + self.compute_product_classes() + + # Execute each "pass". + + # Pre-build-script-impl products... + # Note: currently only supports building for the host. + for host_target in all_host_names: + for product_class in before_impl_product_classes: + if product_class.is_build_script_impl_product(): + continue + if not product_class.is_before_build_script_impl_product(): + continue + # Execute clean, build, test, install + self.execute_product_build_steps(product_class, host_target) + + # Build... + for host_target in all_hosts: + # FIXME: We should only compute these once. + try: + config = HostSpecificConfiguration(host_target.name, self.args) + except argparse.ArgumentError as e: + exit_rejecting_arguments(six.text_type(e)) + print("Building the standard library for: {}".format( + " ".join(config.swift_stdlib_build_targets))) + if config.swift_test_run_targets and ( + self.args.test or self.args.long_test): + print("Running Swift tests for: {}".format( + " ".join(config.swift_test_run_targets))) + if config.swift_benchmark_run_targets and self.args.benchmark: + print("Running Swift benchmarks for: {}".format( + " ".join(config.swift_benchmark_run_targets))) + + for product_class in impl_product_classes: + self._execute_build_action(host_target, product_class) + + # Test... + for host_target in all_hosts: + for product_class in impl_product_classes: + self._execute_test_action(host_target, product_class) + + # Install... + for host_target in all_hosts: + for product_class in impl_product_classes: + self._execute_install_action(host_target, product_class) + + # Core Lipo... + self._execute_merged_host_lipo_core_action() + + # Non-build-script-impl products... + # Note: currently only supports building for the host. + for host_target in [self.args.host_target]: + for product_class in product_classes: + if product_class.is_build_script_impl_product(): + continue + # Execute clean, build, test, install + self.execute_product_build_steps(product_class, host_target) + + # Extract symbols... + for host_target in all_hosts: + self._execute_extract_symbols_action(host_target) + + # Package... + for host_target in all_hosts: + self._execute_package_action(host_target) + + # Lipo... + self._execute_merged_host_lipo_action() + + def _execute_build_action(self, host_target, product_class): + action_name = "{}-{}-build".format(host_target.name, + product_class.product_name()) + self._execute_action(action_name) + + def _execute_test_action(self, host_target, product_class): + action_name = "{}-{}-test".format(host_target.name, + product_class.product_name()) + self._execute_action(action_name) + + def _execute_install_action(self, host_target, product_class): + action_name = "{}-{}-install".format(host_target.name, + product_class.product_name()) + self._execute_action(action_name) + + def _execute_extract_symbols_action(self, host_target): + action_name = "{}-extractsymbols".format(host_target.name) + self._execute_action(action_name) + + def _execute_package_action(self, host_target): + action_name = "{}-package".format(host_target.name) + self._execute_action(action_name) + + def _execute_merged_host_lipo_action(self): + self._execute_action("merged-hosts-lipo") + + def _execute_merged_host_lipo_core_action(self): + self._execute_action("merged-hosts-lipo-core") + + def _execute_action(self, action_name): + shell.call_without_sleeping( + [BUILD_SCRIPT_IMPL_PATH] + self.impl_args + + ["--only-execute", action_name], + env=self.impl_env, echo=self.args.verbose_build) + + def execute_product_build_steps(self, product_class, host_target): + product_source = product_class.product_source_name() + product_name = product_class.product_name() + if product_class.is_swiftpm_unified_build_product(): + build_dir = self.workspace.swiftpm_unified_build_dir( + host_target) + else: + build_dir = self.workspace.build_dir( + host_target, product_name) + product = product_class( + args=self.args, + toolchain=self.toolchain, + source_dir=self.workspace.source_dir(product_source), + build_dir=build_dir) + if product.should_clean(host_target): + print("--- Cleaning %s ---" % product_name) + product.clean(host_target) + if product.should_build(host_target): + print("--- Building %s ---" % product_name) + product.build(host_target) + if product.should_test(host_target): + print("--- Running tests for %s ---" % product_name) + product.test(host_target) + print("--- Finished tests for %s ---" % product_name) + # Install the product if it should be installed specifically, or + # if it should be built and `install_all` is set to True. + # The exception is select before_build_script_impl products + # which set `is_ignore_install_all_product` to True, ensuring + # they are never installed. (e.g. earlySwiftDriver). + if product.should_install(host_target) or \ + (self.install_all and product.should_build(host_target) and + not product.is_ignore_install_all_product()): + print("--- Installing %s ---" % product_name) + product.install(host_target) diff --git a/utils/swift_build_support/swift_build_support/utils.py b/utils/swift_build_support/swift_build_support/utils.py new file mode 100644 index 0000000000000..0316424235b8b --- /dev/null +++ b/utils/swift_build_support/swift_build_support/utils.py @@ -0,0 +1,32 @@ +# ===-- utils.py ---------------------------------------------------------===# +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https:#swift.org/LICENSE.txt for license information +# See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ===---------------------------------------------------------------------===# + +from __future__ import absolute_import, print_function, unicode_literals + +import sys + + +def fatal_error(message, stream=sys.stderr): + """Writes a message to the given stream and exits. By default this + function outputs to stderr. + """ + + stream.write('[{}] ERROR: {}\n'.format(sys.argv[0], message)) + stream.flush() + sys.exit(1) + + +def exit_rejecting_arguments(message, parser=None): + print(message, file=sys.stderr) + if parser: + parser.print_usage(sys.stderr) + sys.exit(2) # 2 is the same as `argparse` error exit code.