Build both host and target toolchains

Resolves #1365.

# Conflicts:
#	utils/build-presets.ini
#	utils/webassembly/build-toolchain.sh
#	utils/webassembly/install-nightly-toolchain.sh

Author: MaxDesiatov

utils/build-presets.ini

@@ -2414,3 +2414,98 @@ no-assertions
 mixin-preset=source_compat_suite_linux_base
 debug
 no-assertions
+
+#===----------------------------------------------------------------------===#
+# Preset for the WebAssembly toolchain and SDK
+#===----------------------------------------------------------------------===#
+
+[preset: webassembly]
+
+release
+cmake-c-launcher=%(C_CXX_LAUNCHER)s
+cmake-cxx-launcher=%(C_CXX_LAUNCHER)s
+skip-build-benchmarks
+llvm-targets-to-build=X86;WebAssembly
+install-destdir=%(INSTALL_DESTDIR)s
+swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
+llvm-install-components=clang
+install-swift
+install-prefix=/%(TOOLCHAIN_NAME)s/usr
+
+[preset: webassembly-host]
+
+mixin-preset=webassembly
+extra-cmake-options=
+    -DSWIFT_BUILD_SOURCEKIT=FALSE
+    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
+    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
+
+llbuild
+swiftpm
+
+install-llvm
+install-swift
+install-llbuild
+install-swiftpm
+
+[preset: webassembly-linux-host]
+
+mixin-preset=webassembly-host
+
+libdispatch
+libicu
+foundation
+xctest
+
+install-libicu
+install-foundation
+install-libdispatch
+install-xctest
+
+skip-test-libicu
+skip-test-foundation
+skip-test-libdispatch
+skip-test-xctest
+
+[preset: webassembly-target]
+
+mixin-preset=webassembly
+wasm
+verbose
+build-stdlib-deployment-targets=wasi-wasm32
+build-swift-dynamic-sdk-overlay=false
+build-swift-dynamic-stdlib=false
+build-swift-static-sdk-overlay
+build-swift-static-stdlib
+stdlib-deployment-targets=wasi-wasm32
+wasi-sdk=%(SOURCE_PATH)s/wasi-sdk
+
+[preset: webassembly-linux-target]
+
+mixin-preset=webassembly-target
+extra-cmake-options=
+    -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
+    -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
+    -DSWIFT_PRIMARY_VARIANT_SDK:STRING=WASI
+    -DSWIFT_PRIMARY_VARIANT_ARCH:STRING=wasm32
+    -DSWIFT_SDKS='WASI;LINUX'
+    -DSWIFT_BUILD_SOURCEKIT=FALSE
+    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
+    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
+    -DCMAKE_AR="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ar"
+    -DCMAKE_RANLIB="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ranlib"
+
+[preset: webassembly-macos-target]
+
+mixin-preset=webassembly-target
+extra-cmake-options=
+    -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
+    -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
+    -DSWIFT_PRIMARY_VARIANT_SDK:STRING=WASI
+    -DSWIFT_PRIMARY_VARIANT_ARCH:STRING=wasm32
+    -DSWIFT_OSX_x86_64_ICU_STATICLIB=TRUE
+    -DSWIFT_BUILD_SOURCEKIT=FALSE
+    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
+    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
+    -DCMAKE_AR='/usr/local/opt/llvm/bin/llvm-ar'
+    -DCMAKE_RANLIB='/usr/local/opt/llvm/bin/llvm-ranlib'

utils/update_checkout/update-checkout-config.json

@@ -59,6 +59,7 @@
                 "swift": "swiftwasm",
                 "cmark": "master",
                 "llbuild": "master",
+                "swift-tools-support-core": "master",
                 "swiftpm": "swiftwasm",
                 "swift-syntax": "master",
                 "swift-stress-tester": "master",
@@ -82,6 +83,7 @@
                 "swift": "swiftwasm-release/5.3",
                 "cmark": "release/5.3",
                 "llbuild": "release/5.3",
+                "swift-tools-support-core": "release/5.3",
                 "swiftpm": "swiftwasm-release/5.3",
                 "swift-syntax": "release/5.3",
                 "swift-stress-tester": "release/5.3",

utils/webassembly/build-swiftpm.sh

@@ -9,11 +9,13 @@ WASI_SDK_PATH=$SOURCE_PATH/wasi-sdk
 case $(uname -s) in
   Darwin)
     OS_SUFFIX=osx
-    BUILD_SCRIPT=$UTILS_PATH/build-mac.sh
+    HOST_PRESET=webassembly-host
+    TARGET_PRESET=webassembly-macos-target
   ;;
   Linux)
     OS_SUFFIX=linux
-    BUILD_SCRIPT=$UTILS_PATH/build-linux.sh
+    HOST_PRESET=webassembly-linux-host
+    TARGET_PRESET=webassembly-linux-target
   ;;
   *)
     echo "Unrecognised platform $(uname -s)"
@@ -35,51 +37,52 @@ BUNDLE_IDENTIFIER="swiftwasm.5.3-${YEAR}${MONTH}${DAY}"
 DISPLAY_NAME_SHORT="Swift for WebAssembly 5.3 Snapshot"
 DISPLAY_NAME="${DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
 
-# Just a hack, spent enough time investigating this, 
-# but somehow `master` is able to work without this `mkdir`.
-mkdir -p $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/clang/10.0.0
-$BUILD_SCRIPT \
-  --install_destdir="$SOURCE_PATH/install" \
-  --installable_package="$INSTALLABLE_PACKAGE" \
-  --install-prefix=/$TOOLCHAIN_NAME/usr \
-  --swift-install-components "autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink" \
-  --llvm-install-components "clang" \
-  --install-swift \
-  --darwin-toolchain-bundle-identifier="${BUNDLE_IDENTIFIER}" \
-  --darwin-toolchain-display-name="${DISPLAY_NAME}" \
-  --darwin-toolchain-display-name-short="${DISPLAY_NAME_SHORT}" \
-  --darwin-toolchain-name="${TOOLCHAIN_NAME}" \
-  --darwin-toolchain-version="${TOOLCHAIN_VERSION}" \
-  --darwin-toolchain-alias="swift" \
-  "$@"
-
-
-NIGHTLY_TOOLCHAIN=$SOURCE_PATH/swift-nightly-toolchain
-if [ ! -e $NIGHTLY_TOOLCHAIN ]; then
-  $UTILS_PATH/install-nightly-toolchain.sh
-fi
-
-TMP_DIR=$(mktemp -d)
-cd $TMP_DIR
-tar xfz $INSTALLABLE_PACKAGE $TOOLCHAIN_NAME
-cd $TMP_DIR/$TOOLCHAIN_NAME
-
-# Merge wasi-sdk and toolchain
-cp -r $WASI_SDK_PATH/lib/clang usr/lib
-cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} usr/bin
-cp -r $WASI_SDK_PATH/share/wasi-sysroot usr/share
-
-# Build SwiftPM and install it into toolchain
-$UTILS_PATH/build-swiftpm.sh $TMP_DIR/$TOOLCHAIN_NAME
+HOST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/host-toolchain-sdk
+HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
+
+# Avoid clang headers symlink issues
+mkdir -p $HOST_TOOLCHAIN_SDK/usr/lib/clang/10.0.0
+
+# Build the host toolchain and SDK first.
+$SOURCE_PATH/swift/utils/build-script \
+  --preset=$HOST_PRESET \
+  INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
+  TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
+  C_CXX_LAUNCHER="$(which sccache)"
+
+# Clean up the host toolchain build directory so that the next
+# `build-script` invocation doesn't pick up wrong CMake config files.
+# For some reason passing `--reconfigure` to `build-script` won't do this.
+rm -rf $SOURCE_PATH/build/Ninja-ReleaseAssert/swift-*
+
+# build the cross-compilled toolchain
+$SOURCE_PATH/swift/utils/build-script \
+  --preset=$TARGET_PRESET \
+  INSTALL_DESTDIR="$SOURCE_PATH/install" \
+  SOURCE_PATH="$SOURCE_PATH" \
+  BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
+  DISPLAY_NAME="${DISPLAY_NAME}" \
+  DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
+  TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
+  TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
+  C_CXX_LAUNCHER="$(which sccache)"
+
+# Merge wasi-sdk and the toolchain
+cp -a $WASI_SDK_PATH/lib/clang $HOST_TOOLCHAIN_SDK/usr/lib
+cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} $HOST_TOOLCHAIN_SDK/usr/bin
+cp -r $WASI_SDK_PATH/share/wasi-sysroot $HOST_TOOLCHAIN_SDK/usr/share
 
 # Replace absolute sysroot path with relative path
-sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
+sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
 
-# Copy nightly-toolchain's host environment stdlib into toolchain
+# Copy the target environment stdlib into the toolchain
 
 # Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
-# but nightly's one is symbolic link, simple copy fails to merge them.
-rsync -a $NIGHTLY_TOOLCHAIN/usr/lib/ $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/ --exclude 'swift/clang'
+# but nightly's one is symbolic link. A simple copy fails to merge them.
+rsync -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/ $HOST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
 
-cd $TMP_DIR
+$UTILS_PATH/build-foundation.sh $HOST_TOOLCHAIN_SDK
+$UTILS_PATH/build-xctest.sh $HOST_TOOLCHAIN_SDK
+
+cd $HOST_TOOLCHAIN_DESTDIR
 tar cfz $PACKAGE_ARTIFACT $TOOLCHAIN_NAME

utils/webassembly/build-toolchain.sh

@@ -0,0 +1,18 @@
+#/bin/bash
+
+set -ex
+
+SOURCE_PATH="$( cd "$(dirname $0)/../../../" && pwd  )"
+
+cd $SOURCE_PATH
+
+WASI_SDK_URL="https://github.com/swiftwasm/wasi-sdk/releases/download/0.2.0-swiftwasm/dist-$2-latest.tgz.zip"
+
+[ ! -e dist-wasi-sdk.tgz.zip ] && \
+  wget -O dist-wasi-sdk.tgz.zip $WASI_SDK_URL
+unzip -u dist-wasi-sdk.tgz.zip -d .
+WASI_SDK_TAR_PATH=$(find . -type f -name "wasi-sdk-*")
+WASI_SDK_FULL_NAME=$(basename $WASI_SDK_TAR_PATH -$1.tar.gz)
+tar xfz $WASI_SDK_TAR_PATH
+rm -rf ./wasi-sdk
+mv $WASI_SDK_FULL_NAME ./wasi-sdk