Revert "Merge pull request #1024 from swiftwasm/maxd/build-host-toolchain"

This reverts commit 88fd32c, reversing
changes made to 04c8aba.

Author: MaxDesiatov

utils/build-presets.ini

@@ -2437,67 +2437,39 @@ no-assertions
 
 [preset: webassembly]
 
+wasm
 release
+verbose
 cmake-c-launcher=%(C_CXX_LAUNCHER)s
 cmake-cxx-launcher=%(C_CXX_LAUNCHER)s
 skip-build-benchmarks
-llvm-targets-to-build=X86;WebAssembly
-install-destdir=%(INSTALL_DESTDIR)s
-swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
-llvm-install-components=clang
-install-swift
-install-prefix=/%(TOOLCHAIN_NAME)s/usr
-
-[preset: webassembly-host]
-
-mixin-preset=webassembly
-extra-cmake-options=
-    -DSWIFT_BUILD_SOURCEKIT=FALSE
-    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
-    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
-
-llbuild
-swiftpm
-
-install-swift
-install-llbuild
-install-swiftpm
-
-[preset: webassembly-linux-host]
-
-mixin-preset=webassembly-host
-
-libdispatch
-libicu
-foundation
-xctest
-
-install-libicu
-install-foundation
-install-libdispatch
-install-xctest
-
-skip-test-libicu
-skip-test-foundation
-skip-test-libdispatch
-skip-test-xctest
-
-[preset: webassembly-target]
-
-mixin-preset=webassembly
-wasm
-verbose
 build-stdlib-deployment-targets=wasi-wasm32
 build-swift-dynamic-sdk-overlay=false
 build-swift-dynamic-stdlib=false
 build-swift-static-sdk-overlay
 build-swift-static-stdlib
+llvm-targets-to-build=X86;WebAssembly
 stdlib-deployment-targets=wasi-wasm32
 wasi-sdk=%(SOURCE_PATH)s/wasi-sdk
 
-[preset: webassembly-linux-target]
+[preset: webassembly-installable]
 
-mixin-preset=webassembly-target
+install_destdir=%(SOURCE_PATH)s/install
+installable_package=%(INSTALLABLE_PACKAGE)s
+install-prefix=/%(TOOLCHAIN_NAME)s/usr
+swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
+llvm-install-components=clang
+install-swift
+darwin-toolchain-bundle-identifier=%(BUNDLE_IDENTIFIER)s
+darwin-toolchain-display-name=%(DISPLAY_NAME)s
+darwin-toolchain-display-name-short=%(DISPLAY_NAME_SHORT)s
+darwin-toolchain-name=%(TOOLCHAIN_NAME)s
+darwin-toolchain-version=%(TOOLCHAIN_VERSION)s
+darwin-toolchain-alias=swift
+
+[preset: webassembly-linux]
+
+mixin-preset=webassembly
 extra-cmake-options=
     -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
     -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
@@ -2510,9 +2482,9 @@ extra-cmake-options=
     -DCMAKE_AR="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ar"
     -DCMAKE_RANLIB="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ranlib"
 
-[preset: webassembly-macos-target]
+[preset: webassembly-macos]
 
-mixin-preset=webassembly-target
+mixin-preset=webassembly
 extra-cmake-options=
     -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
     -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
@@ -2524,3 +2496,15 @@ extra-cmake-options=
     -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
     -DCMAKE_AR='/usr/local/opt/llvm/bin/llvm-ar'
     -DCMAKE_RANLIB='/usr/local/opt/llvm/bin/llvm-ranlib'
+
+[preset: webassembly-linux-installable]
+
+mixin-preset=
+    webassembly-linux
+    webassembly-installable
+
+[preset: webassembly-macos-installable]
+
+mixin-preset=
+    webassembly-macos
+    webassembly-installable

utils/webassembly/build-swiftpm.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -e
+DESTINATION_TOOLCHAIN=$1
+SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
+NIGHTLY_TOOLCHAIN=${SOURCE_PATH}/swift-nightly-toolchain
+
+export PATH=$NIGHTLY_TOOLCHAIN/usr/bin:$PATH
+
+SWIFT_BUILD_FLAGS="-c release"
+SWIFT_BUILD=${NIGHTLY_TOOLCHAIN}/usr/bin/swift-build
+
+build_swiftpm() {
+  local build_flags=$SWIFT_BUILD_FLAGS
+  if [[ "$(uname)" == "Darwin" ]]; then
+    rpath_prefix='@executable_path/../lib/swift/macosx'
+  else
+    rpath_prefix='$ORIGIN/../lib/swift/linux'
+  fi
+  build_flags="${build_flags} -Xlinker -rpath -Xlinker ${rpath_prefix}"
+  cd ${SOURCE_PATH}/swiftpm
+  ${SWIFT_BUILD} ${build_flags}
+}
+
+install_binary() {
+  local src=$1
+  local dest=${DESTINATION_TOOLCHAIN}/usr/bin
+  echo "Installing ${src} to ${dest}"
+  cp ${src} ${dest}
+}
+
+install_runtime_file() {
+  local src=$1
+  local dest=${DESTINATION_TOOLCHAIN}/usr/lib/swift/pm
+  echo "Installing ${src} to ${dest}/{4,4_2}"
+  for runtime in "4" "4_2"
+  do
+    mkdir -p ${dest}/${runtime}
+    cp ${src} ${dest}/${runtime}
+  done
+}
+
+install_swiftpm() {
+  cd ${SOURCE_PATH}/swiftpm
+  local bin_path=$(${SWIFT_BUILD} ${SWIFT_BUILD_FLAGS} --show-bin-path)
+  for binary in ${bin_path}/{swift-build,swift-test,swift-run,swift-package}
+  do
+    install_binary ${binary}
+  done
+
+  if [[ "$(uname)" == "Linux" ]]; then
+    install_runtime_file "${bin_path}/libPackageDescription.so"
+  else
+    install_runtime_file "${bin_path}/libPackageDescription.dylib"
+  fi
+  install_runtime_file "${bin_path}/PackageDescription.swiftmodule"
+  install_runtime_file "${bin_path}/PackageDescription.swiftdoc"
+}
+
+build_swiftpm
+install_swiftpm

utils/webassembly/build-toolchain.sh

@@ -9,13 +9,11 @@ WASI_SDK_PATH=$SOURCE_PATH/wasi-sdk
 case $(uname -s) in
   Darwin)
     OS_SUFFIX=osx
-    HOST_PRESET=webassembly-host
-    TARGET_PRESET=webassembly-macos-target
+    PRESET_NAME=webassembly-macos-installable
   ;;
   Linux)
     OS_SUFFIX=linux
-    HOST_PRESET=webassembly-linux-host
-    TARGET_PRESET=webassembly-linux-target
+    PRESET_NAME=webassembly-linux-installable
   ;;
   *)
     echo "Unrecognised platform $(uname -s)"
@@ -37,54 +35,47 @@ BUNDLE_IDENTIFIER="swiftwasm.${YEAR}${MONTH}${DAY}"
 DISPLAY_NAME_SHORT="Swift for WebAssembly Development Snapshot"
 DISPLAY_NAME="${DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
 
-HOST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/host-toolchain-sdk
-HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
-
-# Avoid clang headers symlink issues
-mkdir -p $HOST_TOOLCHAIN_SDK/usr/lib/clang/10.0.0
-
-# Build the host toolchain and SDK first.
-$SOURCE_PATH/swift/utils/build-script \
-  --preset=$HOST_PRESET \
-  INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
-  TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
-  C_CXX_LAUNCHER="$(which sccache)"
-
-# Clean up the host toolchain build directory so that the next
-# `build-script` invocation doesn't pick up wrong CMake config files.
-# For some reason passing `--reconfigure` to `build-script` won't do this.
-rm -rf $SOURCE_PATH/build/Ninja-ReleaseAssert/swift-*
-
-# build the cross-compilled toolchain
-$SOURCE_PATH/swift/utils/build-script \
-  --preset=$TARGET_PRESET \
-  INSTALL_DESTDIR="$SOURCE_PATH/install" \
+$SOURCE_PATH/swift/utils/build-script --preset=$PRESET_NAME \
   SOURCE_PATH="$SOURCE_PATH" \
+  INSTALLABLE_PACKAGE="$INSTALLABLE_PACKAGE" \
   BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
   DISPLAY_NAME="${DISPLAY_NAME}" \
   DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
   TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
   TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
   C_CXX_LAUNCHER="$(which sccache)"
 
-# Merge wasi-sdk and the toolchain
-cp -a $WASI_SDK_PATH/lib/clang $HOST_TOOLCHAIN_SDK/usr/lib
-cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} $HOST_TOOLCHAIN_SDK/usr/bin
-cp -r $WASI_SDK_PATH/share/wasi-sysroot $HOST_TOOLCHAIN_SDK/usr/share
+
+NIGHTLY_TOOLCHAIN=$SOURCE_PATH/swift-nightly-toolchain
+if [ ! -e $NIGHTLY_TOOLCHAIN ]; then
+  $UTILS_PATH/install-nightly-toolchain.sh
+fi
+
+TMP_DIR=$(mktemp -d)
+cd $TMP_DIR
+tar xfz $INSTALLABLE_PACKAGE $TOOLCHAIN_NAME
+cd $TMP_DIR/$TOOLCHAIN_NAME
+
+# Merge wasi-sdk and toolchain
+cp -r $WASI_SDK_PATH/lib/clang usr/lib
+cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} usr/bin
+cp -r $WASI_SDK_PATH/share/wasi-sysroot usr/share
+
+# Build SwiftPM and install it into toolchain
+$UTILS_PATH/build-swiftpm.sh $TMP_DIR/$TOOLCHAIN_NAME
 
 # Replace absolute sysroot path with relative path
-sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
+sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
 
-# Copy the target environment stdlib into the toolchain
+# Copy nightly-toolchain's host environment stdlib into toolchain
 
 if [[ "$(uname)" == "Linux" ]]; then
-  # Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
-  # but nightly's one is symbolic link. A simple copy fails to merge them.
-  rsync -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/ $HOST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
+  # Avoid to copy usr/lib/swift/clang because our toolchain's one is a directory
+  # but nightly's one is symbolic link, so fail to merge them.
+  rsync -a $NIGHTLY_TOOLCHAIN/usr/lib/ $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/ --exclude 'swift/clang'
 else
-  cp -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift_static $HOST_TOOLCHAIN_SDK/usr/lib/swift_static
-  cp -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi $HOST_TOOLCHAIN_SDK/usr/lib/swift
+  cp -r $NIGHTLY_TOOLCHAIN/usr/lib/swift/macosx $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/swift
 fi
 
-cd $HOST_TOOLCHAIN_DESTDIR
+cd $TMP_DIR
 tar cfz $PACKAGE_ARTIFACT $TOOLCHAIN_NAME

utils/webassembly/install-nightly-toolchain.sh

@@ -0,0 +1,50 @@
+set -x
+
+SOURCE_DIR="$(cd "$(dirname $0)/../../.." && pwd)"
+DESTINATION="${SOURCE_DIR}/swift-nightly-toolchain"
+
+mkdir -p $DESTINATION
+
+install_linux() {
+  WORKSPACE=$(mktemp -d)
+
+  pushd ${WORKSPACE}
+
+  BASE_URL=https://swift.org/builds/development/ubuntu1804
+  export $(/usr/bin/curl ${BASE_URL}/latest-build.yml | grep 'download:' | sed 's/:[^:\/\/]/=/g')
+
+  DOWNLOAD_DIR=$(echo $download | sed "s/-ubuntu18.04.tar.gz//g")
+  DOWNLOAD_URL=${BASE_URL}/${DOWNLOAD_DIR}/${download}
+  /usr/bin/curl ${BASE_URL}/${DOWNLOAD_DIR}/${download} > ${WORKSPACE}/latest_toolchain.tar.gz
+
+  mkdir -p ${WORKSPACE}/latest_toolchain
+  tar xzf ${WORKSPACE}/latest_toolchain.tar.gz -C ${WORKSPACE}/latest_toolchain
+  mv ${WORKSPACE}/latest_toolchain/${DOWNLOAD_DIR}-ubuntu18.04/usr ${DESTINATION}/usr
+
+  popd
+}
+
+install_macos() {
+  WORKSPACE=$(mktemp -d)
+
+  pushd ${WORKSPACE}
+
+  BASE_URL=https://swift.org/builds/development/
+  export $(/usr/bin/curl --silent ${BASE_URL}/xcode/latest-build.yml | grep 'download:' | sed 's/:[^:\/\/]/=/g')
+
+  DOWNLOAD_DIR=$(echo $download | sed "s/-osx.pkg//g")
+  DOWNLOAD_URL=${BASE_URL}/xcode/${DOWNLOAD_DIR}/${download}
+
+  /usr/bin/curl --silent ${DOWNLOAD_URL} > ${WORKSPACE}/latest_xcode_toolchain.pkg
+  pkgutil --expand ${WORKSPACE}/latest_xcode_toolchain.pkg ${WORKSPACE}/latest_xcode_toolchain
+  tar xf latest_xcode_toolchain/${DOWNLOAD_DIR}-osx-package.pkg/Payload
+  mv ${WORKSPACE}/usr ${DESTINATION}/usr
+
+  popd
+}
+
+if [[ "$(uname)" == "Linux" ]]; then
+  install_linux
+else
+  install_macos
+fi