diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c706785..4003b22e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,8 +64,7 @@ jobs: parameters.txt retention-days: 7 - # Build a cross compiler for Linux, targeting Windows. - linux: + linux-stage1: if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw') needs: [prepare] runs-on: ubuntu-22.04 @@ -82,7 +81,82 @@ jobs: # Skip dynamic library dependencies that might make it harder to # run the binaries on other distros (and that have little use within # llvm-mingw). - LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh $(pwd)/install/llvm-mingw + LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh $(pwd)/install/llvm-mingw --stage1 + .github/workflows/store-version.sh install/llvm-mingw/versions.txt + ./test-libcxx-module.sh $(pwd)/install/llvm-mingw + ./run-tests.sh $(pwd)/install/llvm-mingw + cd install + DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-$(uname -m) + NAME=llvm-mingw-stage1-$TAG-ucrt-$DISTRO + mv llvm-mingw $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME + - uses: actions/upload-artifact@v4 + with: + name: linux-stage1-ucrt-x86_64-toolchain + path: | + llvm-mingw-*.tar.xz + retention-days: 7 + + linux-profile: + if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw') + needs: [prepare, linux-stage1] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: linux-stage1-ucrt-x86_64-toolchain + - name: Unpack stage1 Clang + run: | + tar -Jxf llvm-mingw-*.tar.xz + rm llvm-mingw-*.tar.xz + mv llvm-mingw* llvm-mingw-stage1 + - name: Build + env: + LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} + TAG: ${{needs.prepare.outputs.TAG}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} + run: | + # Build LLVM in mostly the same way as it is going to be built in the + # final form. + LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh --profile $(pwd)/llvm-mingw-stage1 + - uses: actions/upload-artifact@v4 + with: + name: profile + path: | + profile.profdata + retention-days: 7 + + # Build a cross compiler for Linux, targeting Windows. + linux: + if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw') + needs: [prepare, linux-stage1, linux-profile] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: linux-stage1-ucrt-x86_64-toolchain + - name: Unpack stage1 Clang + run: | + tar -Jxf llvm-mingw-*.tar.xz + rm llvm-mingw-*.tar.xz + sudo mv llvm-mingw* /opt/llvm-mingw + - uses: actions/download-artifact@v4 + with: + name: profile + - name: Build + env: + LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} + TAG: ${{needs.prepare.outputs.TAG}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} + run: | + # Skip dynamic library dependencies that might make it harder to + # run the binaries on other distros (and that have little use within + # llvm-mingw). + LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw --thinlto --pgo --llvm-only .github/workflows/store-version.sh install/llvm-mingw/versions.txt ./test-libcxx-module.sh $(pwd)/install/llvm-mingw ./run-tests.sh $(pwd)/install/llvm-mingw @@ -102,10 +176,21 @@ jobs: # libraries that were built in the 'linux' step above. linux-cross-aarch64: if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw') - needs: [linux, prepare] + needs: [linux-stage1, linux-profile, prepare] runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: linux-stage1-ucrt-x86_64-toolchain + - name: Unpack stage1 Clang + run: | + tar -Jxf llvm-mingw-*.tar.xz + rm llvm-mingw-*.tar.xz + sudo mv llvm-mingw* /opt/llvm-mingw + - uses: actions/download-artifact@v4 + with: + name: profile - name: Build env: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} @@ -113,19 +198,11 @@ jobs: SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} run: | sudo apt-get update && sudo apt-get install g++-aarch64-linux-gnu - ./build-all.sh $(pwd)/install/llvm-mingw --no-runtimes --host=aarch64-linux-gnu + ./build-all.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw --no-runtimes --host=aarch64-linux-gnu --thinlto --pgo .github/workflows/store-version.sh install/llvm-mingw/versions.txt - - uses: actions/download-artifact@v4 - with: - name: linux-ucrt-x86_64-toolchain - - name: Unpack native toolchain - run: | - tar -Jxf llvm-mingw-*.tar.xz - rm llvm-mingw-*.tar.xz - mv llvm-mingw* llvm-mingw-native - name: Assemble the cross-built toolchain run: | - ./prepare-cross-toolchain-unix.sh $(pwd)/llvm-mingw-native $(pwd)/install/llvm-mingw + ./prepare-cross-toolchain-unix.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw - name: Test using the cross-built assembled toolchain run: | sudo apt-get update && sudo apt-get install qemu-user-static libc6-arm64-cross libstdc++6-arm64-cross @@ -183,13 +260,53 @@ jobs: llvm-mingw-linux.tar.xz retention-days: 7 + macos-stage1: + if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw') + needs: [prepare] + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: Build + env: + LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} + TAG: ${{needs.prepare.outputs.TAG}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} + run: | + # Disable zstd and python. Both are available on the runners, but + # installed with homebrew, and only available in the native (arm64) + # form. + LLVM_CMAKEFLAGS="-DLLVM_ENABLE_ZSTD=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh $(pwd)/install/llvm --stage1 --llvm-only + .github/workflows/store-version.sh install/llvm/versions.txt + cd install + NAME=llvm-$TAG-macos-$(uname -m) + mv llvm $NAME + gtar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME + - uses: actions/upload-artifact@v4 + with: + name: macos-llvm + path: | + llvm-*.tar.xz + retention-days: 7 + # Build a cross compiler for macOS, targeting Windows. macos: if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw') - needs: [prepare] + needs: [prepare, macos-stage1, linux-profile] runs-on: macos-latest steps: - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: macos-llvm + - name: Unpack stage1 Clang + run: | + tar -Jxf llvm-*.tar.xz + rm llvm-*.tar.xz + mv llvm* $HOME/llvm + - uses: actions/download-artifact@v4 + with: + name: profile - name: Build env: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} @@ -203,7 +320,7 @@ jobs: # form. Therefore, autodetection will pick them up, but linking # universal binaries fails as those libraries are unavailable in the # other (x86_64) architecture form. - MACOS_REDIST=1 LLVM_CMAKEFLAGS="-DLLVM_ENABLE_ZSTD=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh $(pwd)/install/llvm-mingw + MACOS_REDIST=1 LLVM_CMAKEFLAGS="-DLLVM_ENABLE_ZSTD=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh $HOME/llvm $(pwd)/install/llvm-mingw --thinlto --pgo .github/workflows/store-version.sh install/llvm-mingw/versions.txt ./test-libcxx-module.sh $(pwd)/install/llvm-mingw ./run-tests.sh $(pwd)/install/llvm-mingw @@ -280,7 +397,7 @@ jobs: # architectures). The binaries built here match actual releases quite closely. linux-cross-windows: if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw') - needs: [linux, prepare] + needs: [linux-stage1, linux-profile, prepare] runs-on: ubuntu-latest strategy: fail-fast: false @@ -293,7 +410,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: linux-${{matrix.crt}}-x86_64-toolchain + name: linux-stage1-${{matrix.crt}}-x86_64-toolchain - name: Unpack cross toolchain run: | tar -Jxf llvm-mingw-*.tar.xz @@ -301,6 +418,9 @@ jobs: sudo mv llvm-mingw* /opt/llvm-mingw echo /opt/llvm-mingw/bin >> $GITHUB_PATH - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: profile - name: Build env: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} @@ -311,7 +431,7 @@ jobs: BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} run: | sudo apt-get update && sudo apt-get install libltdl-dev swig autoconf-archive - ./build-cross-tools.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw ${{matrix.arch}} --with-python + ./build-cross-tools.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw ${{matrix.arch}} --with-python --thinlto --pgo .github/workflows/store-version.sh install/llvm-mingw/versions.txt cd install NAME=llvm-mingw-$TAG-${{matrix.crt}}-${{matrix.arch}} @@ -689,6 +809,9 @@ jobs: run: | rm -rf linux-asserts* rm -rf msys2* + rm -rf linux-stage1-* + rm -rf macos-llvm* + rm -rf profile* mv *-toolchain/*.zip *-toolchain/*.tar.xz . - name: Upload binaries env: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 411a20be..c992bb54 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -64,18 +64,11 @@ jobs: docker-build: needs: [prepare] runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - { file: Dockerfile, key: amd64 } - - { file: Dockerfile.toolchain, platforms: linux/arm64, key: arm64 } steps: - uses: actions/checkout@v4 with: ref: ${{inputs.ref}} - name: Download toolchain - if: ${{matrix.file == 'Dockerfile.toolchain'}} uses: dawidd6/action-download-artifact@v10 with: workflow: build.yml @@ -86,7 +79,6 @@ jobs: name_is_regexp: true path: toolchain - name: Set up QEMU - if: ${{matrix.file == 'Dockerfile.toolchain'}} uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -98,42 +90,11 @@ jobs: password: ${{secrets.DOCKER_PASSWORD}} - name: Build Docker images uses: docker/build-push-action@v5 - id: build with: context: . - platforms: ${{matrix.platforms}} + platforms: linux/amd64,linux/arm64 push: ${{inputs.push}} - file: ./${{matrix.file}} - outputs: | - type=image,name=mstorsjo/llvm-mingw,push-by-digest=true,name-canonical=true - - name: Write outputs for later steps - uses: cloudposse/github-action-matrix-outputs-write@main - id: out - with: - matrix-step-name: ${{github.job}} - matrix-key: ${{matrix.key}} - outputs: |- - digest: ${{steps.build.outputs.digest}} - - docker-create: - needs: [docker-build, prepare] - runs-on: ubuntu-latest - if: ${{inputs.push}} - steps: - - uses: cloudposse/github-action-matrix-outputs-read@main - id: read - with: - matrix-step-name: docker-build - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{vars.DOCKER_USERNAME}} - password: ${{secrets.DOCKER_PASSWORD}} - - name: Create final image - run: | - set -x - for tag in latest ${{needs.prepare.outputs.TAG}}; do - docker buildx imagetools create -t mstorsjo/llvm-mingw:$tag mstorsjo/llvm-mingw@${{fromJson(steps.read.outputs.result).digest.amd64}} mstorsjo/llvm-mingw@${{fromJson(steps.read.outputs.result).digest.arm64}} - done + file: ./Dockerfile.toolchain + tags: | + mstorsjo/llvm-mingw:latest + mstorsjo/llvm-mingw:${{needs.prepare.outputs.TAG}} diff --git a/.github/workflows/msvcrt.yml b/.github/workflows/msvcrt.yml index 9fb12761..ff0856e4 100644 --- a/.github/workflows/msvcrt.yml +++ b/.github/workflows/msvcrt.yml @@ -106,6 +106,14 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{inputs.ref}} + - name: Download profile + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build.yml + workflow_conclusion: success + ref: ${{inputs.ref}} + event: push + name: profile - name: Build env: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} @@ -114,7 +122,7 @@ jobs: TAG: ${{needs.prepare.outputs.TAG}} run: | sudo apt-get update && sudo apt-get install libltdl-dev swig autoconf-archive - ./build-cross-tools.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw ${{matrix.arch}} --with-python + ./build-cross-tools.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw ${{matrix.arch}} --with-python --thinlto --pgo .github/workflows/store-version.sh install/llvm-mingw/versions.txt cd install NAME=llvm-mingw-$TAG-${{matrix.crt}}-${{matrix.arch}} diff --git a/build-all.sh b/build-all.sh index 42c21e04..39bfa676 100755 --- a/build-all.sh +++ b/build-all.sh @@ -24,7 +24,7 @@ HOST_ARGS="" while [ $# -gt 0 ]; do case "$1" in - --enable-asserts) + --enable-asserts|--disable-dylib|--with-clang|--thinlto) LLVM_ARGS="$LLVM_ARGS $1" ;; --host-clang|--host-clang=*) @@ -36,9 +36,6 @@ while [ $# -gt 0 ]; do LLVM_ARGS="$LLVM_ARGS $1" FULL_LLVM=1 ;; - --disable-dylib) - LLVM_ARGS="$LLVM_ARGS $1" - ;; --disable-lldb) LLVM_ARGS="$LLVM_ARGS $1" NO_LLDB=1 @@ -76,18 +73,58 @@ while [ $# -gt 0 ]; do --clean-runtimes) CLEAN_RUNTIMES=1 ;; + --llvm-only) + LLVM_ONLY=1 + ;; + --stage1) + STAGE1=1 + LLVM_ARGS="$LLVM_ARGS --disable-lldb --disable-clang-tools-extra" + NO_LLDB=1 + ;; + --profile|--profile=*) + case "$1" in + --profile=*) + INSTRUMENTATION="=${1#*=}" + ;; + esac + PROFILE=1 + LLVM_ARGS="$LLVM_ARGS --disable-lldb --disable-clang-tools-extra --with-clang --disable-dylib --instrumented$INSTRUMENTATION" + NO_LLDB=1 + LLVM_ONLY=1 + ;; + --pgo|--pgo=*) + PGO=1 + LLVM_ARGS="$LLVM_ARGS --with-clang $1" + ;; + --full-pgo|--full-pgo=*) + case "$1" in + --full-pgo=*) + INSTRUMENTATION="=${1#*=}" + ;; + esac + PGO=1 + FULL_PGO=1 + ;; *) if [ -n "$PREFIX" ]; then - echo Unrecognized parameter $1 - exit 1 + if [ -n "$PREFIX_PGO" ]; then + echo Unrecognized parameter $1 + exit 1 + fi + PREFIX_PGO="$1" + else + PREFIX="$1" fi - PREFIX="$1" ;; esac shift done if [ -z "$PREFIX" ]; then - echo "$0 [--host-clang[=clang]] [--enable-asserts] [--disable-dylib] [--full-llvm] [--disable-lldb] [--disable-lldb-mi] [--disable-clang-tools-extra] [--host=triple] [--with-default-win32-winnt=0x601] [--with-default-msvcrt=ucrt] [--enable-cfguard|--disable-cfguard] [--no-runtimes] [--no-tools] [--wipe-runtimes] [--clean-runtimes] dest" + echo "$0 [--host-clang[=clang]] [--enable-asserts] [--disable-dylib] [--with-clang] [--thinlto] [--full-llvm] [--disable-lldb] [--disable-lldb-mi] [--disable-clang-tools-extra] [--host=triple] [--with-default-win32-winnt=0x601] [--with-default-msvcrt=ucrt] [--enable-cfguard|--disable-cfguard] [--no-runtimes] [--llvm-only] [--no-tools] [--wipe-runtimes] [--clean-runtimes] [--stage1] [--profile[=type]] [--pgo[=profile]] [--full-pgo[=type]] dest [pgo-dest]" + exit 1 +fi +if [ -n "$PREFIX_PGO" ] && [ -z "$PGO" ] && [ -z "$FULL_PGO" ]; then + echo Unrecognized parameter $1 exit 1 fi @@ -102,15 +139,83 @@ if [ -n "${HOST_CLANG}" ] && [ "${CFGUARD_ARGS}" = "--enable-cfguard" ]; then "${HOST_CLANG}" -c -x c -o - - -Werror -mguard=cf /dev/null 2>/dev/null || CFGUARD_ARGS="--disable-cfguard" fi +if [ -n "$FULL_PGO" ]; then + if [ -z "$PREFIX_PGO" ]; then + echo Must provide a second destination for a PGO build + exit 1 + fi + ./build-all.sh "$PREFIX" --stage1 $LLVM_ARGS $MINGW_ARGS $CFGUARD_ARGS + unset COMPILER_LAUNCHER + ./build-all.sh "$PREFIX" --profile$INSTRUMENTATION $LLVM_ARGS + ./build-all.sh "$PREFIX" "$PREFIX_PGO" --thinlto --pgo --llvm-only $LLVM_ARGS + # If one already has a usable profile, one could also do the following + # two steps only: + # ./build-all.sh "$PREFIX" --stage1 --llvm-only + # ./build-all.sh "$PREFIX" "$PREFIX_PGO" --pgo + exit 0 +fi + +if [ -n "$PROFILE" ]; then + export PATH=$PREFIX/bin:$PATH + STAGE1_PREFIX=$PREFIX + PREFIX=/tmp/dummy-prefix +elif [ -n "$PGO" ]; then + if [ -z "$PREFIX_PGO" ]; then + echo Must provide a second destination for a PGO build + exit 1 + fi + export PATH=$PREFIX/bin:$PATH + STAGE1_PREFIX=$PREFIX + PREFIX=$PREFIX_PGO + + if [ -n "$LLVM_ONLY" ] && [ "$PREFIX" != "$STAGE1_PREFIX" ] ; then + # Only rebuilding LLVM, not any runtimes. Copy the stage1 toolchain + # and rebuild LLVM on top of it. + rm -rf $PREFIX + mkdir -p "$(dirname "$PREFIX")" + cp -a "$STAGE1_PREFIX" "$PREFIX" + # Remove the native Linux/macOS runtimes which aren't needed in + # the final distribution. + rm -rf "$PREFIX"/lib/clang/*/lib/darwin + rm -rf "$PREFIX"/lib/clang/*/lib/linux + fi +fi + +if [ "$(uname)" = "Darwin" ]; then + if [ -n "$PROFILE" ] || [ -n "$PGO" ]; then + # Using a custom Clang, which doesn't find the SDK automatically. + # CMake sets this automatically, but if using the compiler directly, + # this is needed. (If compilation uses "cc" or "gcc", it will miss + # the stage1 llvm-mingw toolchain and use the system compiler anyway.) + export SDKROOT=$(xcrun --show-sdk-path) + fi +fi + if [ -z "$NO_TOOLS" ]; then if [ -z "${HOST_CLANG}" ]; then ./build-llvm.sh $PREFIX $LLVM_ARGS $HOST_ARGS + if [ -n "$PROFILE" ]; then + ./pgo-training.sh llvm-project/llvm/build-instrumented $STAGE1_PREFIX + exit 0 + fi if [ -z "$NO_LLDB" ] && [ -z "$NO_LLDB_MI" ]; then ./build-lldb-mi.sh $PREFIX $HOST_ARGS fi if [ -z "$FULL_LLVM" ]; then ./strip-llvm.sh $PREFIX $HOST_ARGS fi + if [ -n "$STAGE1" ]; then + if [ "$(uname)" = "Darwin" ]; then + ./build-llvm.sh $PREFIX --macos-native-tools + fi + # Build runtimes. On Linux, this is needed for profiling. + # On macOS, it is also needed for OS availability helpers like + # __isPlatformVersionAtLeast. + ./build-compiler-rt.sh --native $PREFIX + fi + fi + if [ -n "$LLVM_ONLY" ]; then + exit 0 fi ./install-wrappers.sh $PREFIX $HOST_ARGS ${HOST_CLANG:+--host-clang=$HOST_CLANG} ./build-mingw-w64-tools.sh $PREFIX $HOST_ARGS diff --git a/build-compiler-rt.sh b/build-compiler-rt.sh index d7f3e3d7..89c8d722 100755 --- a/build-compiler-rt.sh +++ b/build-compiler-rt.sh @@ -40,13 +40,16 @@ while [ $# -gt 0 ]; do elif [ "$1" = "--disable-cfguard" ]; then CFGUARD_CFLAGS= ENABLE_CFGUARD= + elif [ "$1" = "--native" ]; then + NATIVE=1 + SRC_DIR=.. else PREFIX="$1" fi shift done if [ -z "$PREFIX" ]; then - echo "$0 [--build-sanitizers] [--enable-cfguard|--disable-cfguard] dest" + echo "$0 [--build-sanitizers] [--enable-cfguard|--disable-cfguard] [--native] dest" exit 1 fi if [ -n "$SANITIZERS" ] && [ -n "$ENABLE_CFGUARD" ]; then @@ -59,8 +62,7 @@ export PATH="$PREFIX/bin:$PATH" : ${ARCHS:=${TOOLCHAIN_ARCHS-i686 x86_64 armv7 aarch64}} -ANY_ARCH=$(echo $ARCHS | awk '{print $1}') -CLANG_RESOURCE_DIR="$("$PREFIX/bin/$ANY_ARCH-w64-mingw32-clang" --print-resource-dir)" +CLANG_RESOURCE_DIR="$("$PREFIX/bin/clang" --print-resource-dir)" if [ ! -d llvm-project/compiler-rt ] || [ -n "$SYNC" ]; then CHECKOUT_ONLY=1 ./build-llvm.sh @@ -91,6 +93,34 @@ if [ -h "$CLANG_RESOURCE_DIR/include" ]; then INSTALL_PREFIX="$WORKDIR/install" fi +if [ -n "$NATIVE" ]; then + [ -z "$CLEAN" ] || rm -rf build-native + mkdir -p build-native + cd build-native + [ -n "$NO_RECONF" ] || rm -rf CMake* + cmake \ + ${CMAKE_GENERATOR+-G} "$CMAKE_GENERATOR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$CLANG_RESOURCE_DIR" \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_CONFIG_PATH="" \ + -DCMAKE_FIND_ROOT_PATH=$PREFIX \ + -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY \ + -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY \ + -DCOMPILER_RT_USE_LIBCXX=OFF \ + $SRC_DIR + cmake --build . ${CORES:+-j${CORES}} + cmake --install . --prefix "$INSTALL_PREFIX" + + if [ "$INSTALL_PREFIX" != "$CLANG_RESOURCE_DIR" ]; then + # symlink to system headers - skip copy + rm -rf "$INSTALL_PREFIX/include" + + cp -r "$INSTALL_PREFIX/." $CLANG_RESOURCE_DIR + fi + exit 0 +fi for arch in $ARCHS; do [ -z "$CLEAN" ] || rm -rf build-$arch$BUILD_SUFFIX diff --git a/build-cross-tools.sh b/build-cross-tools.sh index 1aa94804..fd28ec71 100755 --- a/build-cross-tools.sh +++ b/build-cross-tools.sh @@ -44,6 +44,9 @@ while [ $# -gt 0 ]; do --disable-make) NO_MAKE=1 ;; + --thinlto|--lto|--pgo*) + LLVM_ARGS="$LLVM_ARGS $1" + ;; *) if [ -z "$NATIVE" ]; then NATIVE="$1" @@ -60,7 +63,7 @@ while [ $# -gt 0 ]; do shift done if [ -z "$CROSS_ARCH" ]; then - echo $0 native prefix arch [--with-python] [--disable-lldb] [--disable-lldb-mi] [--disable-clang-tool-extra] [--disable-mingw-w64-tools] [--disable-make] + echo $0 native prefix arch [--with-python] [--disable-lldb] [--disable-lldb-mi] [--disable-clang-tool-extra] [--disable-mingw-w64-tools] [--disable-make] [--thinlto] [--lto] [--pgo[=profile]] exit 1 fi diff --git a/build-lldb-mi.sh b/build-lldb-mi.sh index 7c228213..1dd6b13d 100755 --- a/build-lldb-mi.sh +++ b/build-lldb-mi.sh @@ -65,7 +65,7 @@ else esac fi -export LLVM_DIR="$PREFIX" +LLVM_DIR="$PREFIX" # Try to find/guess the builddir under the llvm buildtree next by. # If LLVM was built without LLVM_INSTALL_TOOLCHAIN_ONLY, and the LLVM @@ -75,12 +75,32 @@ export LLVM_DIR="$PREFIX" LLVM_SRC="$(pwd)/llvm-project/llvm" if [ -d "$LLVM_SRC" ]; then SUFFIX=${HOST+-}$HOST - for base in build build-asserts; do - if [ -d "$LLVM_SRC/$base$SUFFIX" ]; then - export LLVM_DIR="$LLVM_SRC/$base$SUFFIX" - break + DIRS="" + cd llvm-project/llvm + for dir in build*$SUFFIX; do + if [ -z "$SUFFIX" ]; then + case $dir in + *linux*|*mingw32*) + continue + ;; + esac + fi + if [ -d "$dir" ]; then + DIRS="$DIRS $dir" fi done + if [ -n "$DIRS" ]; then + dir="$(ls -td $DIRS | head -1)" + LLVM_DIR="$LLVM_SRC/$dir" + echo Using $LLVM_DIR as LLVM build dir + break + else + # No build directory found; this is ok if the installed prefix is a + # full (development) install of LLVM. Warn that we didn't find what + # we were looking for. + echo Warning, did not find a suitable LLVM build dir, assuming $PREFIX contains LLVM development files >&2 + fi + cd ../.. fi if [ -n "$HOST" ]; then @@ -101,14 +121,14 @@ if [ -n "$HOST" ]; then exit 1 ;; esac - - CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH=$LLVM_DIR" - CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER" - CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY" - CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY" - CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY" fi +CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH=$LLVM_DIR" +CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER" +CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY" +CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY" +CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY" + if [ -n "$MACOS_REDIST" ]; then : ${MACOS_REDIST_ARCHS:=arm64 x86_64} : ${MACOS_REDIST_VERSION:=10.12} diff --git a/build-llvm.sh b/build-llvm.sh index de2c4a43..9f21b713 100755 --- a/build-llvm.sh +++ b/build-llvm.sh @@ -24,6 +24,7 @@ LINK_DYLIB=ON ASSERTSSUFFIX="" LLDB=ON CLANG_TOOLS_EXTRA=ON +INSTRUMENTED=OFF while [ $# -gt 0 ]; do case "$1" in @@ -35,9 +36,9 @@ while [ $# -gt 0 ]; do ASSERTS=ON ASSERTSSUFFIX="-asserts" ;; - --stage2) - STAGE2=1 - BUILDDIR="$BUILDDIR-stage2" + --with-clang) + WITH_CLANG=1 + BUILDDIR="$BUILDDIR-withclang" ;; --thinlto) LTO="thin" @@ -68,6 +69,31 @@ while [ $# -gt 0 ]; do --no-llvm-tool-reuse) NO_LLVM_TOOL_REUSE=1 ;; + --macos-native-tools) + MACOS_NATIVE_TOOLS=1 + ;; + --instrumented|--instrumented=*) + INSTRUMENTED="${1#--instrumented}" + INSTRUMENTED="${INSTRUMENTED#=}" + INSTRUMENTED="${INSTRUMENTED:-Frontend}" + : ${LLVM_PROFILE_DATA_DIR:=/tmp/llvm-profile} + # A fixed BUILDDIR is set at the end for this case. + ;; + --pgo|--pgo=*) + case "$1" in + --pgo=*) + LLVM_PROFDATA_FILE="${1#--pgo}" + LLVM_PROFDATA_FILE="${LLVM_PROFDATA_FILE#=}" + ;; + esac + LLVM_PROFDATA_FILE="${LLVM_PROFDATA_FILE:-profile.profdata}" + if [ ! -e "$LLVM_PROFDATA_FILE" ]; then + echo Profile \"$LLVM_PROFDATA_FILE\" not found + exit 1 + fi + LLVM_PROFDATA_FILE="$(cd "$(dirname "$LLVM_PROFDATA_FILE")" && pwd)/$(basename "$LLVM_PROFDATA_FILE")" + BUILDDIR="$BUILDDIR-pgo" + ;; *) PREFIX="$1" ;; @@ -77,12 +103,14 @@ done BUILDDIR="$BUILDDIR$ASSERTSSUFFIX" if [ -z "$CHECKOUT_ONLY" ]; then if [ -z "$PREFIX" ]; then - echo $0 [--enable-asserts] [--stage2] [--thinlto] [--lto] [--disable-dylib] [--full-llvm] [--with-python] [--disable-lldb] [--disable-clang-tools-extra] [--host=triple] dest + echo $0 [--enable-asserts] [--with-clang] [--thinlto] [--lto] [--instrumented[=type]] [--pgo[=profile]] [--disable-dylib] [--full-llvm] [--with-python] [--disable-lldb] [--disable-clang-tools-extra] [--host=triple] [--macos-native-tools] dest exit 1 fi - mkdir -p "$PREFIX" - PREFIX="$(cd "$PREFIX" && pwd)" + if [ -z "$INSTRUMENTED" ]; then + mkdir -p "$PREFIX" + PREFIX="$(cd "$PREFIX" && pwd)" + fi fi if [ ! -d llvm-project ]; then @@ -157,9 +185,18 @@ CMAKEFLAGS="$LLVM_CMAKEFLAGS" if [ -n "$HOST" ]; then ARCH="${HOST%%-*}" - CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_C_COMPILER=$HOST-gcc" - CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_CXX_COMPILER=$HOST-g++" - CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_SYSTEM_PROCESSOR=$ARCH" + + if [ -n "$WITH_CLANG" ]; then + CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_C_COMPILER=clang" + CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_CXX_COMPILER=clang++" + CMAKEFLAGS="$CMAKEFLAGS -DLLVM_USE_LINKER=lld" + CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_C_COMPILER_TARGET=$HOST" + CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_CXX_COMPILER_TARGET=$HOST" + else + CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_C_COMPILER=$HOST-gcc" + CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_CXX_COMPILER=$HOST-g++" + CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_SYSTEM_PROCESSOR=$ARCH" + fi case $HOST in *-mingw32) CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_SYSTEM_NAME=Windows" @@ -220,9 +257,9 @@ if [ -n "$HOST" ]; then CMAKEFLAGS="$CMAKEFLAGS -DPython3_INCLUDE_DIRS=$PYTHON_INCLUDE_DIR" CMAKEFLAGS="$CMAKEFLAGS -DPython3_LIBRARIES=$PYTHON_LIB" fi -elif [ -n "$STAGE2" ]; then - # Build using an earlier built and installed clang in the target directory - export PATH="$PREFIX/bin:$PATH" +elif [ -n "$WITH_CLANG" ]; then + # Build using clang and lld (from $PATH), rather than the system default + # tools. CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_C_COMPILER=clang" CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_CXX_COMPILER=clang++" CMAKEFLAGS="$CMAKEFLAGS -DLLVM_USE_LINKER=lld" @@ -289,6 +326,25 @@ if [ -z "$HOST" ] && [ "$(uname)" = "Darwin" ]; then # This silences a cmake warning. CMAKEFLAGS="$CMAKEFLAGS -DLLDB_USE_SYSTEM_DEBUGSERVER=ON" fi + if [ -n "$WITH_CLANG" ] && [ -n "$LTO" ]; then + # If doing LTO, we need to make sure other related tools are used. + # CMAKE_LIBTOOL is not a standard cmake tool, but an LLVM specific + # thing. It defaults to looking up the tool with xcrun rather than + # looking in paths first. + # If building for multiple architectures at once + # (CMAKE_OSX_ARCHITECTURES), we also need to provide a tool named + # "lipo" in the PATH (this is invoked by Clang directly, so we can't + # specify it here unless we pass in the option "-fuse-lipo="). If + # the LLVM CMake build wasn't using libtool, we would also need to + # specify LLVM_AR and LLVM_RANLIB. + CMAKEFLAGS="$CMAKEFLAGS -DCMAKE_LIBTOOL=$(command -v llvm-libtool-darwin)" + fi +fi + +if [ "$INSTRUMENTED" != "OFF" ]; then + # For instrumented build, use a hardcoded builddir that we can + # locate, and don't install the built files. + BUILDDIR="build-instrumented" fi TOOLCHAIN_ONLY=ON @@ -309,6 +365,24 @@ fi [ -z "$CLEAN" ] || rm -rf $BUILDDIR mkdir -p $BUILDDIR cd $BUILDDIR + +if [ -n "$MACOS_NATIVE_TOOLS" ]; then + # Build tools needed for targeting macOS with LTO. + # + # The install-(-stripped) targets are unavailable for + # tools that are excluded due to LLVM_INSTALL_TOOLCHAIN_ONLY=ON and + # LLVM_TOOLCHAIN_TOOLS, so install those manually. + cmake --build . --target llvm-lipo --target llvm-libtool-darwin + # Install ld64.lld, required for -fuse-ld=lld + cmake --install . --strip --component lld + # Install llvm-libtool-darwin and lipo, needed for building with LTO. + # See the comment further above for more details about this. + cp bin/llvm-lipo bin/llvm-libtool-darwin $PREFIX/bin + strip $PREFIX/bin/llvm-lipo $PREFIX/bin/llvm-libtool-darwin + ln -sf llvm-lipo $PREFIX/bin/lipo + exit 0 +fi + [ -n "$NO_RECONF" ] || rm -rf CMake* cmake \ ${CMAKE_GENERATOR+-G} "$CMAKE_GENERATOR" \ @@ -321,10 +395,19 @@ cmake \ -DLLVM_LINK_LLVM_DYLIB=$LINK_DYLIB \ -DLLVM_TOOLCHAIN_TOOLS="llvm-ar;llvm-ranlib;llvm-objdump;llvm-rc;llvm-cvtres;llvm-nm;llvm-strings;llvm-readobj;llvm-dlltool;llvm-pdbutil;llvm-objcopy;llvm-strip;llvm-cov;llvm-profdata;llvm-addr2line;llvm-symbolizer;llvm-windres;llvm-ml;llvm-readelf;llvm-size;llvm-cxxfilt;llvm-lib" \ ${HOST+-DLLVM_HOST_TRIPLE=$HOST} \ + -DLLVM_BUILD_INSTRUMENTED=$INSTRUMENTED \ + ${LLVM_PROFILE_DATA_DIR+-DLLVM_PROFILE_DATA_DIR=$LLVM_PROFILE_DATA_DIR} \ + ${LLVM_PROFDATA_FILE+-DLLVM_PROFDATA_FILE=$LLVM_PROFDATA_FILE} \ $CMAKEFLAGS \ .. -cmake --build . ${CORES:+-j${CORES}} -cmake --install . --strip +if [ "$INSTRUMENTED" != "OFF" ]; then + # For instrumented builds, don't install the built files (so $PREFIX + # is entirely unused). + cmake --build . ${CORES:+-j${CORES}} --target clang --target lld +else + cmake --build . ${CORES:+-j${CORES}} + cmake --install . --strip -cp ../LICENSE.TXT $PREFIX + cp ../LICENSE.TXT $PREFIX +fi diff --git a/pgo-training.make b/pgo-training.make new file mode 100755 index 00000000..b26bc029 --- /dev/null +++ b/pgo-training.make @@ -0,0 +1,57 @@ +#!/bin/sh +# +# Copyright (c) 2025 Martin Storsjo +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +CFLAGS = --sysroot=$(STAGE1) -resource-dir=$(shell $(STAGE1)/bin/clang --print-resource-dir) --config-user-dir=$(STAGE1)/bin +CC = $(PREFIX)/bin/clang +CXX = $(PREFIX)/bin/clang++ + +hello-exception-opt-%.exe: test/hello-exception.cpp + $(CXX) -target $*-w64-mingw32 $(CFLAGS) $+ -o $@ -O3 + +hello-exception-%.exe: test/hello-exception.cpp + $(CXX) -target $*-w64-mingw32 $(CFLAGS) $+ -o $@ + +sqlite-opt-%.exe: $(SQLITE)/sqlite3.c $(SQLITE)/shell.c + $(CC) -target $*-w64-mingw32 $(CFLAGS) $+ -o $@ -O3 + +sqlite-%.exe: $(SQLITE)/sqlite3.c $(SQLITE)/shell.c + $(CC) -target $*-w64-mingw32 $(CFLAGS) $+ -o $@ + +LIBCXXTEST = llvm-project/libcxx/test/std/algorithms/alg.sorting/alg.sort/sort/sort.pass.cpp + +libcxxtest-opt-%.exe: $(LIBCXXTEST) + $(CXX) -target $*-w64-mingw32 $(CFLAGS) $+ -o $@ -Illvm-project/libcxx/test/support -O3 + +libcxxtest-%.exe: $(LIBCXXTEST) + $(CXX) -target $*-w64-mingw32 $(CFLAGS) $+ -o $@ -Illvm-project/libcxx/test/support + +ARCHS ?= i686 x86_64 armv7 aarch64 + +TARGETS = hello-exception hello-exception-opt + +ifneq ($(SQLITE),) +TARGETS += sqlite sqlite-opt +endif +ifneq ($(wildcard $(LIBCXXTEST)),) +TARGETS += libcxxtest libcxxtest-opt +endif + +ALLTARGETS = $(foreach arch, $(ARCHS), $(foreach target, $(TARGETS), $(target)-$(arch).exe)) + +all: $(ALLTARGETS) + +clean: + rm -f $(ALLTARGETS) diff --git a/pgo-training.sh b/pgo-training.sh new file mode 100755 index 00000000..918e0e55 --- /dev/null +++ b/pgo-training.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# +# Copyright (c) 2025 Martin Storsjo +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +set -e + +: ${SQLITE_VERSION:=3490200} +: ${SQLITE_YEAR:=2025} + +: ${LLVM_PROFILE_DATA_DIR:=/tmp/llvm-profile} +: ${LLVM_PROFDATA_FILE:=profile.profdata} + +if [ $# -lt 2 ]; then + echo $0 build stage1 + exit 1 +fi +PREFIX="$1" +STAGE1="$2" +PREFIX="$(cd "$PREFIX" && pwd)" + +MAKE=make +if command -v gmake >/dev/null; then + MAKE=gmake +fi + +: ${CORES:=$(nproc 2>/dev/null)} +: ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)} +: ${CORES:=4} +: ${ARCHS:=${TOOLCHAIN_ARCHS-i686 x86_64 armv7 aarch64}} + +download() { + if command -v curl >/dev/null; then + curl -LO "$1" + else + wget "$1" + fi +} + +SQLITE=sqlite-amalgamation-$SQLITE_VERSION +if [ ! -d $SQLITE ]; then + download https://sqlite.org/$SQLITE_YEAR/sqlite-amalgamation-$SQLITE_VERSION.zip + unzip sqlite-amalgamation-$SQLITE_VERSION.zip +fi + +rm -rf "$LLVM_PROFILE_DATA_DIR" +$MAKE -f pgo-training.make PREFIX=$PREFIX STAGE1=$STAGE1 SQLITE=$SQLITE clean +$MAKE -f pgo-training.make PREFIX=$PREFIX STAGE1=$STAGE1 SQLITE=$SQLITE -j$CORES + +rm -f "$LLVM_PROFDATA_FILE" +$STAGE1/bin/llvm-profdata merge -output "$LLVM_PROFDATA_FILE" $LLVM_PROFILE_DATA_DIR/*.profraw +rm -rf "$LLVM_PROFILE_DATA_DIR"