[WASM] Refactoring build script for debuggability

kateinoigakukun · kateinoigakukun · commit f8bb90b01b07 · 2020-09-22T11:42:13.000Z
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -96,7 +96,7 @@ jobs:
           name: packaging-scripts
           path: utils/webassembly
       - name: Pack test results
-        run: tar cJf swift-test-results.tar.gz ../build/*/swift-macosx-x86_64/swift-test-results
+        run: tar cJf swift-test-results.tar.gz ../target-build/*/swift-macosx-x86_64/swift-test-results
       - name: Upload test results
         uses: actions/upload-artifact@v1
         with:
diff --git a/utils/build-presets.ini b/utils/build-presets.ini
@@ -2442,8 +2442,9 @@ skip-build-benchmarks
 llvm-targets-to-build=X86;AArch64;WebAssembly
 install-destdir=%(INSTALL_DESTDIR)s
 swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
-llvm-install-components=clang
+llvm-install-components=clang;compiler-rt;lld;llvm-ar;llvm-ranlib
 install-swift
+install-llvm
 install-prefix=/%(TOOLCHAIN_NAME)s/usr
 
 [preset: webassembly-host]
@@ -2457,7 +2458,6 @@ extra-cmake-options=
 llbuild
 swiftpm
 
-install-llvm
 install-swift
 install-llbuild
 install-swiftpm
diff --git a/utils/build-script-impl b/utils/build-script-impl
@@ -1610,6 +1610,11 @@ for host in "${ALL_HOSTS[@]}"; do
 
                 llvm_enable_projects=("clang")
 
+                # wasm: lld is necessary to sync compiler llvm version and lld llvm version
+                if [[ ! "${SKIP_BUILD_LLD}" ]]; then
+                    llvm_enable_projects+=("lld")
+                fi
+
                 if [[ ! "${SKIP_BUILD_COMPILER_RT}" && ! $(is_cross_tools_host ${host}) ]]; then
                     llvm_enable_projects+=("compiler-rt")
                 fi
diff --git a/utils/webassembly/build-foundation.sh b/utils/webassembly/build-foundation.sh
@@ -3,11 +3,7 @@ set -ex
 DESTINATION_TOOLCHAIN=$1
 SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
 
-# Remove host CoreFoundation (which can be different from the target) headers
-# to avoid shadowing the wasm32 target CoreFoundation
-rm -rf $DESTINATION_TOOLCHAIN/usr/lib/swift/CoreFoundation
-
-FOUNDATION_BUILD="$SOURCE_PATH/build/Ninja-ReleaseAssert/foundation-wasi-wasm32"
+FOUNDATION_BUILD="$SOURCE_PATH/target-build/Ninja-ReleaseAssert/foundation-wasi-wasm32"
 
 mkdir -p $FOUNDATION_BUILD
 cd $FOUNDATION_BUILD
@@ -19,15 +15,8 @@ cmake -G Ninja \
   -DWASI_SDK_PATH="$SOURCE_PATH/wasi-sdk" \
   -DICU_ROOT="$SOURCE_PATH/icu_out" \
   -DBUILD_SHARED_LIBS=OFF \
+  -DCMAKE_Swift_COMPILER_FORCED=ON \
   "${SOURCE_PATH}/swift-corelibs-foundation"
   
 ninja -v
 ninja -v install
-
-# On macOS the target CoreFoundation shadows the CoreFoundation suppplied by Xcode.
-# On Linux though there's no system CoreFoundation, its headers have to be shipped
-# in the installable archive and serve for both host and target.
-if [[ "$(uname)" == "Darwin" ]]; then
-  mv $DESTINATION_TOOLCHAIN/usr/lib/swift/CoreFoundation \
-    $DESTINATION_TOOLCHAIN/usr/lib/swift/wasi/wasm32/CoreFoundation
-fi
diff --git a/utils/webassembly/build-toolchain.sh b/utils/webassembly/build-toolchain.sh
@@ -11,11 +11,13 @@ case $(uname -s) in
     OS_SUFFIX=osx
     HOST_PRESET=webassembly-host
     TARGET_PRESET=webassembly-macos-target
+    HOST_SUFFIX=macosx-x86_64
   ;;
   Linux)
     OS_SUFFIX=linux
     HOST_PRESET=webassembly-linux-host
     TARGET_PRESET=webassembly-linux-target
+    HOST_SUFFIX=linux-x86_64
   ;;
   *)
     echo "Unrecognised platform $(uname -s)"
@@ -37,54 +39,88 @@ BUNDLE_IDENTIFIER="swiftwasm.5.3-${YEAR}${MONTH}${DAY}"
 DISPLAY_NAME_SHORT="Swift for WebAssembly 5.3 Snapshot"
 DISPLAY_NAME="${DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
 
+DIST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/dist-toolchain-sdk
 HOST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/host-toolchain-sdk
-HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
+TARGET_TOOLCHAIN_DESTDIR=$SOURCE_PATH/target-toolchain-sdk
 
-# Avoid clang headers symlink issues
-mkdir -p $HOST_TOOLCHAIN_SDK/usr/lib/clang/10.0.0
-
-# Build the host toolchain and SDK first.
-$SOURCE_PATH/swift/utils/build-script \
-  --preset=$HOST_PRESET \
-  INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
-  TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
-  C_CXX_LAUNCHER="$(which sccache)"
-
-# Clean up the host toolchain build directory so that the next
-# `build-script` invocation doesn't pick up wrong CMake config files.
-# For some reason passing `--reconfigure` to `build-script` won't do this.
-rm -rf $SOURCE_PATH/build/Ninja-ReleaseAssert/swift-*
-
-# build the cross-compilled toolchain
-$SOURCE_PATH/swift/utils/build-script \
-  --preset=$TARGET_PRESET \
-  INSTALL_DESTDIR="$SOURCE_PATH/install" \
-  SOURCE_PATH="$SOURCE_PATH" \
-  BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
-  DISPLAY_NAME="${DISPLAY_NAME}" \
-  DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
-  TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
-  TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
-  C_CXX_LAUNCHER="$(which sccache)"
-
-# Merge wasi-sdk and the toolchain
-cp -a $WASI_SDK_PATH/lib/clang $HOST_TOOLCHAIN_SDK/usr/lib
-cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} $HOST_TOOLCHAIN_SDK/usr/bin
-cp -r $WASI_SDK_PATH/share/wasi-sysroot $HOST_TOOLCHAIN_SDK/usr/share
-
-# Replace absolute sysroot path with relative path
-sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/wasi.modulemap
-
-# Copy the target environment stdlib into the toolchain
-
-# Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
-# but nightly's one is symbolic link. A simple copy fails to merge them.
-rsync -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/ $HOST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
-rsync -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/bin/ $HOST_TOOLCHAIN_SDK/usr/bin/
-
-# FIXME: avoid building foundation for now
-$UTILS_PATH/build-foundation.sh $HOST_TOOLCHAIN_SDK
-$UTILS_PATH/build-xctest.sh $HOST_TOOLCHAIN_SDK
-
-cd $HOST_TOOLCHAIN_DESTDIR
+DIST_TOOLCHAIN_SDK=$DIST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
+HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
+TARGET_TOOLCHAIN_SDK=$TARGET_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
+
+
+HOST_BUILD_ROOT=$SOURCE_PATH/host-build
+TARGET_BUILD_ROOT=$SOURCE_PATH/target-build
+HOST_BUILD_DIR=$HOST_BUILD_ROOT/Ninja-ReleaseAssert
+TARGET_BUILD_DIR=$TARGET_BUILD_ROOT/Ninja-ReleaseAssert
+
+build_host_toolchain() {
+  # Avoid clang headers symlink issues
+  mkdir -p $HOST_TOOLCHAIN_SDK/usr/lib/clang/10.0.0
+  # Build the host toolchain and SDK first.
+  env SWIFT_BUILD_ROOT="$HOST_BUILD_ROOT" \
+    $SOURCE_PATH/swift/utils/build-script \
+    --preset=$HOST_PRESET \
+    --build-dir="$HOST_BUILD_DIR" \
+    INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
+    TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
+    C_CXX_LAUNCHER="$(which sccache)"
+}
+
+build_target_toolchain() {
+  mkdir -p $HOST_BUILD_DIR/
+  # Copy the host build dir to reuse it.
+  if [[ ! -e "$HOST_BUILD_DIR/llvm-$HOST_SUFFIX" ]]; then
+    cp -r "$HOST_BUILD_DIR/llvm-$HOST_SUFFIX" "$TARGET_BUILD_DIR/llvm-$HOST_SUFFIX"
+    # Clean up compiler-rt dir to cross compile it for host and wasm32
+    (cd "$TARGET_BUILD_DIR/llvm-$HOST_SUFFIX" && ninja compiler-rt-clear)
+  fi
+
+  # Avoid clang headers symlink issues
+  mkdir -p $TARGET_TOOLCHAIN_SDK/usr/lib/clang/10.0.0
+
+  # build the cross-compilled toolchain
+  env SWIFT_BUILD_ROOT="$TARGET_BUILD_ROOT" \
+    $SOURCE_PATH/swift/utils/build-script \
+    --preset=$TARGET_PRESET --reconfigure \
+    --build-dir="$TARGET_BUILD_DIR" \
+    INSTALL_DESTDIR="$TARGET_TOOLCHAIN_DESTDIR" \
+    SOURCE_PATH="$SOURCE_PATH" \
+    BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
+    DISPLAY_NAME="${DISPLAY_NAME}" \
+    DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
+    TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
+    TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
+    C_CXX_LAUNCHER="$(which sccache)"
+
+  $UTILS_PATH/build-foundation.sh $TARGET_TOOLCHAIN_SDK
+  $UTILS_PATH/build-xctest.sh $TARGET_TOOLCHAIN_SDK
+
+}
+
+merge_toolchains() {
+  rm -rf "$DIST_TOOLCHAIN_DESTDIR"
+  # Copy the base host toolchain
+  cp -r "$HOST_TOOLCHAIN_DESTDIR" "$DIST_TOOLCHAIN_DESTDIR"
+
+  # Merge wasi-sdk and the toolchain
+  cp -a $WASI_SDK_PATH/lib/clang $DIST_TOOLCHAIN_SDK/usr/lib
+  cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} $DIST_TOOLCHAIN_SDK/usr/bin
+  cp -r $WASI_SDK_PATH/share/wasi-sysroot $DIST_TOOLCHAIN_SDK/usr/share
+
+  # Copy the target environment stdlib into the toolchain
+  # Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
+  # but nightly's one is symbolic link. A simple copy fails to merge them.
+  rsync -v -a $TARGET_TOOLCHAIN_SDK/usr/lib/ $DIST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
+  rsync -v -a $TARGET_TOOLCHAIN_SDK/usr/bin/ $DIST_TOOLCHAIN_SDK/usr/bin/
+
+  # Replace absolute sysroot path with relative path
+  sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $DIST_TOOLCHAIN_SDK/usr/lib/swift/wasi/wasm32/wasi.modulemap
+}
+
+build_host_toolchain
+build_target_toolchain
+
+merge_toolchains
+
+cd $DIST_TOOLCHAIN_DESTDIR
 tar cfz $PACKAGE_ARTIFACT $TOOLCHAIN_NAME
diff --git a/utils/webassembly/build-xctest.sh b/utils/webassembly/build-xctest.sh
@@ -3,7 +3,7 @@ set -ex
 DESTINATION_TOOLCHAIN=$1
 SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
 
-BUILD_DIR="$SOURCE_PATH/build/Ninja-ReleaseAssert/xctest-wasi-wasm32"
+BUILD_DIR="$SOURCE_PATH/target-build/Ninja-ReleaseAssert/xctest-wasi-wasm32"
 
 mkdir -p $BUILD_DIR
 cd $BUILD_DIR
@@ -14,6 +14,7 @@ cmake -G Ninja \
   -DCMAKE_TOOLCHAIN_FILE="$SOURCE_PATH/swift/utils/webassembly/toolchain-wasi.cmake" \
   -DWASI_SDK_PATH="$SOURCE_PATH/wasi-sdk" \
   -DBUILD_SHARED_LIBS=OFF \
+  -DCMAKE_Swift_COMPILER_FORCED=ON \
   -DSWIFT_FOUNDATION_PATH=$DESTINATION_TOOLCHAIN/usr/lib/swift/wasi/wasm32 \
   "${SOURCE_PATH}/swift-corelibs-xctest"
   
diff --git a/utils/webassembly/ci.sh b/utils/webassembly/ci.sh
@@ -13,7 +13,7 @@ fi
 
 BUILD_SCRIPT=$UTILS_PATH/build-toolchain.sh
 RUN_TEST_BIN=$SWIFT_PATH/utils/run-test
-BUILD_DIR=$SOURCE_PATH/build/Ninja-ReleaseAssert
+TARGET_BUILD_DIR=$SOURCE_PATH/target-build/Ninja-ReleaseAssert
 
 $DEPENDENCIES_SCRIPT
 
@@ -29,10 +29,10 @@ $BUILD_SCRIPT $FLAGS
 
 if [[ "$(uname)" == "Darwin" ]]; then
   # workaround: host target test directory is necessary to use run-test
-  mkdir -p $BUILD_DIR/swift-macosx-x86_64/test-macosx-x86_64
+  mkdir -p $TARGET_BUILD_DIR/swift-macosx-x86_64/test-macosx-x86_64
 fi
 
-$RUN_TEST_BIN --build-dir $BUILD_DIR --target wasi-wasm32 test/stdlib/
+$RUN_TEST_BIN --build-dir $TARGET_BUILD_DIR --target wasi-wasm32 test/stdlib/
 
 if [[ "$(uname)" == "Linux" ]]; then
   echo "Skip running test suites for Linux"
