From d77d78b18c066bec58ceda496a964cd91aed08a2 Mon Sep 17 00:00:00 2001 From: oligamiq Date: Sun, 1 Sep 2024 16:00:58 +0900 Subject: [PATCH 01/12] Compile rustc for wasm15 with llvm --- .gitmodules | 4 +- Cargo.lock | 7 +- comment.txt | 22 +++ compiler/rustc_fs_util/src/lib.rs | 2 +- compiler/rustc_llvm/Cargo.toml | 2 +- compiler/rustc_llvm/build.rs | 14 ++ .../src/spec/targets/wasm32_wasip1_threads.rs | 3 +- config.llvm.toml | 40 ++++ src/bootstrap/src/core/build_steps/llvm.rs | 172 ++++++++++++++++++ 9 files changed, 259 insertions(+), 7 deletions(-) create mode 100644 config.llvm.toml diff --git a/.gitmodules b/.gitmodules index 2082ec9ef455f..928fb4cd67b3c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -32,8 +32,8 @@ shallow = true [submodule "src/llvm-project"] path = src/llvm-project - url = https://github.com/rust-lang/llvm-project.git - branch = rustc/19.1-2024-07-30 + url = https://github.com/YoWASP/llvm-project + branch = main+wasm shallow = true [submodule "src/doc/embedded-book"] path = src/doc/embedded-book diff --git a/Cargo.lock b/Cargo.lock index f01db1dbf34ef..39a555a62c940 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -407,9 +407,12 @@ version = "0.1.0" [[package]] name = "cc" -version = "1.0.105" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437" +checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" diff --git a/comment.txt b/comment.txt index cf4db963835cc..fbd796f866c5c 100644 --- a/comment.txt +++ b/comment.txt @@ -16,3 +16,25 @@ $ gcc -fuse-ld=lld tmp/rmeta*/lib.rmeta tmp/rust_out.* dist/lib/rustlib/x86_64-u $ ./rust_out Hello World! ``` + +use LLVM +Install example: +WASI_SDK_PATH=`pwd`/wasi-sdk-24.0-x86_64-linux WASI_SYSROOT=`pwd`/wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot ./x.py install + +If you just want to run it, https://github.com/oligamiq/rust_wasm/tree/main/rustc_llvm +``` +$ mkdir tmp +$ echo 'fn main() { println!("Hello World!"); }' | wasmtime run -Sthreads=y -Spreview2=n --dir tmp::/ --dir dist --env RUST_MIN_STACK=16777216 dist/bin/rustc.wasm - --sysroot dist --target wasm32-wasip1-threads -Csave-temps +$ gcc -fuse-ld=lld tmp/rmeta*/lib.rmeta tmp/rust_out.*.o dist/lib/rustlib/x86_64-unknown-linux-gnu/lib/lib*.rlib -o rust_out +$ ./rust_out +Hello World! +``` + +to Wasi +``` +$ mkdir tmp +$ echo 'fn main() { println!("Hello World!"); }' | wasmtime run -Sthreads=y -Spreview2=n --dir tmp::/ --dir dist --env RUST_MIN_STACK=16777216 dist/bin/rustc.wasm - --sysroot dist --target wasm32-wasip1-threads -Csave-temps +$ wasi-sdk-24.0-x86_64-linux/bin/wasm-ld --shared-memory --max-memory=1073741824 --import-memory --export __main_void -z stack-size=1048576 --stack-first --allow-undefined --no-demangle --import-memory --export-memory --shared-memory dist/lib/rustlib/wasm32-wasip1-threads/lib/self-contained/crt1-command.o tmp/rust_out.*.o dist/lib/rustlib/wasm32-wasip1-threads/lib/lib*.rlib -L dist/lib/rustlib/wasm32-wasip1-threads/lib/self-contained -lc -o rust_out.wasm +$ wasmtime run -Sthreads=y rust_out.wasm +Hello World! +``` diff --git a/compiler/rustc_fs_util/src/lib.rs b/compiler/rustc_fs_util/src/lib.rs index 69a9da530b853..95e77e67d958b 100644 --- a/compiler/rustc_fs_util/src/lib.rs +++ b/compiler/rustc_fs_util/src/lib.rs @@ -84,7 +84,7 @@ pub fn path_to_c_string(p: &Path) -> CString { let p: &OsStr = p.as_ref(); CString::new(p.as_bytes()).unwrap() } -#[cfg(windows)] +#[cfg(any(windows, target_os = "wasi"))] pub fn path_to_c_string(p: &Path) -> CString { CString::new(p.to_str().unwrap()).unwrap() } diff --git a/compiler/rustc_llvm/Cargo.toml b/compiler/rustc_llvm/Cargo.toml index 1f74aaf9965a7..b248bd2de00d4 100644 --- a/compiler/rustc_llvm/Cargo.toml +++ b/compiler/rustc_llvm/Cargo.toml @@ -10,5 +10,5 @@ libc = "0.2.73" [build-dependencies] # tidy-alphabetical-start -cc = "=1.0.105" # FIXME(cc): pinned to keep support for VS2013 +cc = "1.1.15" # tidy-alphabetical-end diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index f092110a324ec..7ca2dd29fe7db 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -102,6 +102,10 @@ fn output(cmd: &mut Command) -> String { } fn main() { + if env::var("TARGET").expect("TARGET was not set").contains("wasi") { + std::env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set"); + } + for component in REQUIRED_COMPONENTS.iter().chain(OPTIONAL_COMPONENTS.iter()) { println!("cargo:rustc-check-cfg=cfg(llvm_component,values(\"{component}\"))"); } @@ -201,6 +205,16 @@ fn main() { cfg.define("NDEBUG", None); } + + if target.contains("wasi") { + // ref src/bootstrap/src/core/build_steps/llvm.rs + + let wasi_sysroot = env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set"); + cfg.compiler(format!("{wasi_sysroot}/../../bin/{target}-clang++")); + cfg.flag("-pthread"); + cfg.flag("-D_WASI_EMULATED_MMAN"); + } + rerun_if_changed_anything_in_dir(Path::new("llvm-wrapper")); cfg.file("llvm-wrapper/PassWrapper.cpp") .file("llvm-wrapper/RustWrapper.cpp") diff --git a/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs b/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs index c06d7c879d42e..5b914f227bbe7 100644 --- a/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs +++ b/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs @@ -17,7 +17,7 @@ pub(crate) fn target() -> Target { options.add_pre_link_args( LinkerFlavor::WasmLld(Cc::No), - &["--import-memory", "--export-memory", "--shared-memory", "-Wl,--max-memory=1073741824"], + &["--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", "-lwasi-emulated-mman"], ); options.add_pre_link_args( LinkerFlavor::WasmLld(Cc::Yes), @@ -27,6 +27,7 @@ pub(crate) fn target() -> Target { "-Wl,--export-memory,", "-Wl,--shared-memory", "-Wl,--max-memory=1073741824", + "-lwasi-emulated-mman", ], ); diff --git a/config.llvm.toml b/config.llvm.toml new file mode 100644 index 0000000000000..c847474f7c973 --- /dev/null +++ b/config.llvm.toml @@ -0,0 +1,40 @@ +# Includes one of the default files in src/bootstrap/defaults +profile = "compiler" +change-id = 9999999 + +[rust] +codegen-backends = ["llvm"] +deny-warnings = false +llvm-bitcode-linker = false + +[llvm] +cflags = "-march=native" +cxxflags = "-march=native" +static-libstdcpp = true +ninja = false +download-ci-llvm = false +link-shared = false +# thin-lto = true + +[build] +docs = false +extended = false +tools = [] +# host = ["wasm32-wasip1-threads", "x86_64-unknown-linux-gnu"] +host = ["wasm32-wasip1-threads"] +target = ["x86_64-unknown-linux-gnu", "wasm32-wasip1-threads"] +# target = ["wasm32-wasip1-threads"] +cargo-native-static = true + +[install] +prefix = "dist" +sysconfdir = "etc" + +[target.'wasm32-wasip1-threads'] +wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot" +# codegen-backends = ["cranelift"] +linker = "wasi-sdk-24.0-x86_64-linux/bin/clang" +codegen-backends = ["llvm"] + +[target.'x86_64-unknown-linux-gnu'] +cc = "gcc" diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index d8752d037618e..af773f1e8ce3f 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -515,6 +515,178 @@ impl Step for Llvm { return res; } + if target.contains("wasi") { + let wasi_sysroot = env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set"); + let wasi_sdk_path = std::path::Path::new(&wasi_sysroot).join("../../").canonicalize().expect("invalid WASI_SYSROOT"); + let wasi_sysroot = format!("--sysroot={wasi_sysroot}"); + let wasi_sdk_path = wasi_sdk_path.to_str().expect("invalid WASI_SYSROOT"); + let wasi_target = target.triple.to_string(); + let wasi_cflags = String::from(""); + let wasi_ldflags = String::from(""); + let wasi_target_llvm = target.triple.to_string(); + let wasi_cflags_llvm = format!("{wasi_cflags} -pthread"); + let wasi_ldflags_llvm = wasi_ldflags; + // LLVM has some (unreachable in our configuration) calls to mmap. + let wasi_cflags_llvm = format!("{wasi_cflags_llvm} -D_WASI_EMULATED_MMAN"); + let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -lwasi-emulated-mman"); + // Depending on the code being compiled, both Clang and LLD can consume unbounded amounts of memory. + let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -Wl,--max-memory=4294967296"); + // Compiling C++ code requires a lot of stack space and can overflow and corrupt the heap. + // (For example, `#include ` alone does it in a build with the default stack size.) + let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -Wl,-z,stack-size=1048576 -Wl,--stack-first"); + // Some of the host APIs that are statically required by LLVM (notably threading) are dynamically + // never used. An LTO build removes imports of these APIs, simplifying deployment + let wasi_cflags_llvm = format!("{wasi_cflags_llvm} -flto"); + let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -flto -Wl,--strip-all"); + + // We need two toolchain files: one for the compiler itself (which needs threads at the moment since + // -DLLVM_ENABLE_THREADS=OFF is kind of broken), and one for the runtime libs. + cfg.define("WASI", "TRUE") + .define("CMAKE_SYSTEM_NAME", "Generic") + .define("CMAKE_SYSTEM_VERSION", "1") + .define("CMAKE_SYSTEM_PROCESSOR", "wasm32") + .define("CMAKE_EXECUTABLE_SUFFIX", ".wasm") + .define("CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER") + .define("CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", "ONLY") + .define("CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", "ONLY") + .define("CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", "ONLY") + .define("CMAKE_C_COMPILER", format!("{wasi_sdk_path}/bin/{wasi_target}-clang")) + .define("CMAKE_CXX_COMPILER", format!("{wasi_sdk_path}/bin/{wasi_target}-clang++")) + .define("CMAKE_LINKER", format!("{wasi_sdk_path}/bin/wasm-ld")) + .define("CMAKE_AR", format!("{wasi_sdk_path}/bin/ar")) + .define("CMAKE_RANLIB", format!("{wasi_sdk_path}/bin/ranlib")) + .define("CMAKE_C_COMPILER_TARGET", &wasi_target_llvm) + .define("CMAKE_CXX_COMPILER_TARGET", &wasi_target_llvm) + .define("CMAKE_C_FLAGS", format!("{wasi_sysroot} {wasi_cflags_llvm}")) + .define("CMAKE_CXX_FLAGS", format!("{wasi_sysroot} {wasi_cflags_llvm}")) + .define("CMAKE_EXE_LINKER_FLAGS", wasi_ldflags_llvm) + .define("LLVM_BUILD_SHARED_LIBS", "OFF") + .define("LLVM_ENABLE_PIC", "OFF") + .define("LLVM_BUILD_STATIC", "ON") + // .define("LLVM_ENABLE_THREADS", "OFF") + .define("LLVM_ENABLE_THREADS", "ON") + .define("LLVM_BUILD_RUNTIME", "OFF") + .define("LLVM_BUILD_TOOLS", "OFF") + .define("LLVM_INCLUDE_UTILS", "OFF") + .define("LLVM_BUILD_UTILS", "OFF") + .define("LLVM_INCLUDE_RUNTIMES", "OFF") + .define("LLVM_INCLUDE_EXAMPLES", "OFF") + .define("LLVM_INCLUDE_TESTS", "OFF") + .define("LLVM_INCLUDE_BENCHMARKS", "OFF") + .define("LLVM_INCLUDE_DOCS", "OFF") + .define("LLVM_TOOL_BUGPOINT_BUILD", "OFF") + .define("LLVM_TOOL_BUGPOINT_PASSES_BUILD", "OFF") + .define("LLVM_TOOL_DSYMUTIL_BUILD", "OFF") + .define("LLVM_TOOL_DXIL_DIS_BUILD", "OFF") + .define("LLVM_TOOL_GOLD_BUILD", "OFF") + .define("LLVM_TOOL_LLC_BUILD", "OFF") + .define("LLVM_TOOL_LLI_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_AR_BUILD", "ON") + // .define("LLVM_TOOL_LLVM_AS_BUILD", "ON") + .define("LLVM_TOOL_LLVM_AS_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_BCANALYZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_CAT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_CFI_VERIFY_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_CONFIG_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_COV_BUILD", "ON") + .define("LLVM_TOOL_LLVM_CVTRES_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_CXXDUMP_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_CXXFILT_BUILD", "ON") + .define("LLVM_TOOL_LLVM_CXXMAP_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_C_TEST_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DEBUGINFOD_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DEBUGINFOD_FIND_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DEBUGINFO_ANALYZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DIFF_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DIS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DIS_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DLANG_DEMANGLE_FUZZER_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_DRIVER_BUILD", "ON") + // .define("LLVM_TOOL_LLVM_DWARFDUMP_BUILD", "ON") + .define("LLVM_TOOL_LLVM_DWARFUTIL_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DWP_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_EXEGESIS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_EXTRACT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_GSYMUTIL_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_IFS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_ISEL_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_ITANIUM_DEMANGLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_JITLINK_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_JITLISTENER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_LIBTOOL_DARWIN_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_LINK_BUILD", "ON") + .define("LLVM_TOOL_LLVM_LIPO_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_LTO2_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_LTO_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MCA_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MC_ASSEMBLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MC_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MC_DISASSEMBLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MICROSOFT_DEMANGLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_ML_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MODEXTRACT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_NM_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_OBJCOPY_BUILD", "ON") + // .define("LLVM_TOOL_LLVM_OBJDUMP_BUILD", "ON") + .define("LLVM_TOOL_LLVM_OPT_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_OPT_REPORT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_PDBUTIL_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_PROFDATA_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_PROFGEN_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_RC_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_READOBJ_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_READTAPI_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_REDUCE_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_REMARKUTIL_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_RTDYLD_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_RUST_DEMANGLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SHLIB_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SIM_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SIZE_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SPLIT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_STRESS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_STRINGS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SYMBOLIZER_BUILD", "ON") + .define("LLVM_TOOL_LLVM_TLI_CHECKER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_UNDNAME_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_XRAY_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_YAML_NUMERIC_PARSER_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_YAML_PARSER_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LTO_BUILD", "OFF") + .define("LLVM_TOOL_OBJ2YAML_BUILD", "OFF") + .define("LLVM_TOOL_OPT_BUILD", "OFF") + .define("LLVM_TOOL_OPT_VIEWER_BUILD", "OFF") + .define("LLVM_TOOL_REDUCE_CHUNK_LIST_BUILD", "OFF") + .define("LLVM_TOOL_REMARKS_SHLIB_BUILD", "OFF") + .define("LLVM_TOOL_SANCOV_BUILD", "OFF") + .define("LLVM_TOOL_SANSTATS_BUILD", "OFF") + .define("LLVM_TOOL_SPIRV_TOOLS_BUILD", "OFF") + .define("LLVM_TOOL_VERIFY_USELISTORDER_BUILD", "OFF") + .define("LLVM_TOOL_VFABI_DEMANGLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_XCODE_TOOLCHAIN_BUILD", "OFF") + .define("LLVM_TOOL_YAML2OBJ_BUILD", "OFF") + // .define("LLVM_ENABLE_PROJECTS", "clang;lld") + .define("LLVM_ENABLE_PROJECTS", "") + // .define("CLANG_ENABLE_ARCMT", "OFF") + // .define("CLANG_ENABLE_STATIC_ANALYZER", "OFF") + // .define("CLANG_INCLUDE_TESTS", "OFF") + // .define("CLANG_BUILD_TOOLS", "OFF") + // .define("CLANG_TOOL_CLANG_SCAN_DEPS_BUILD", "OFF") + // .define("CLANG_TOOL_CLANG_INSTALLAPI_BUILD", "OFF") + // .define("CLANG_BUILD_EXAMPLES", "OFF") + // .define("CLANG_INCLUDE_DOCS", "OFF") + // .define("CLANG_LINKS_TO_CREATE", "clang;clang++") + // .define("CLANG_LINKS_TO_CREATE", "") + .define("LLD_BUILD_TOOLS", "OFF") + .define("CMAKE_BUILD_TYPE", "MinSizeRel") + .define("HAVE_DLOPEN", ""); + } else { + cfg.define("LLVM_TOOL_LLVM_CONFIG_BUILD", "ON") + .define("LLVM_BUILD_TOOLS", "ON"); + } + cfg.build(); // Helper to find the name of LLVM's shared library on darwin and linux. From 6273b05d44fc067efa6f63e547e877e8b5672ddb Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:39:51 +0200 Subject: [PATCH 02/12] Some fixes --- compiler/rustc_codegen_ssa/Cargo.toml | 2 +- config.llvm.toml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 3ab4cd0a0f53c..1223c8a49c627 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" ar_archive_writer = "0.4.2" arrayvec = { version = "0.7", default-features = false } bitflags = "2.4.1" -cc = "=1.0.105" # FIXME(cc): pinned to keep support for VS2013 +cc = "1.1.15" either = "1.5.0" itertools = "0.12" jobserver = "0.1.28" diff --git a/config.llvm.toml b/config.llvm.toml index c847474f7c973..b8d5239e0fee0 100644 --- a/config.llvm.toml +++ b/config.llvm.toml @@ -6,6 +6,8 @@ change-id = 9999999 codegen-backends = ["llvm"] deny-warnings = false llvm-bitcode-linker = false +debug = false +debuginfo-level = 0 [llvm] cflags = "-march=native" From 2e091417b9f1a0265af5dc73fa935e721c60e07d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:50:01 +0200 Subject: [PATCH 03/12] Fix llvm-project submodule --- src/llvm-project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llvm-project b/src/llvm-project index 4b8d29c585687..69bd8547e9cc1 160000 --- a/src/llvm-project +++ b/src/llvm-project @@ -1 +1 @@ -Subproject commit 4b8d29c585687084bbcf21471e04f279d1eddc0a +Subproject commit 69bd8547e9cc1268ca41339c98b2b17d13d69be3 From 22ceada4dea6aa945c6049c95089b6552f15d237 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:42:12 +0200 Subject: [PATCH 04/12] Another fix --- compiler/rustc_fs_util/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_fs_util/src/lib.rs b/compiler/rustc_fs_util/src/lib.rs index 95e77e67d958b..b7f267b4bbd11 100644 --- a/compiler/rustc_fs_util/src/lib.rs +++ b/compiler/rustc_fs_util/src/lib.rs @@ -1,4 +1,4 @@ -#[cfg(any(unix, windows))] +#[cfg(any(unix, windows, target_os = "wasi"))] use std::ffi::CString; use std::path::{absolute, Path, PathBuf}; use std::{fs, io}; From ee85060db0d9c11da50e4aa6a5c2e431e8894f0b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:59:33 +0200 Subject: [PATCH 05/12] Fix linking libc++ on wasi --- compiler/rustc_llvm/build.rs | 1 + config.llvm.toml | 4 +++- src/bootstrap/src/core/build_steps/compile.rs | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index 7ca2dd29fe7db..ff35568cfc8db 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -377,6 +377,7 @@ fn main() { || target.contains("freebsd") || target.contains("windows-gnullvm") || target.contains("aix") + || target.contains("wasi") { "c++" } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() { diff --git a/config.llvm.toml b/config.llvm.toml index b8d5239e0fee0..7b5081f1a9075 100644 --- a/config.llvm.toml +++ b/config.llvm.toml @@ -33,8 +33,10 @@ prefix = "dist" sysconfdir = "etc" [target.'wasm32-wasip1-threads'] -wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot" +wasi-root = "/home/bjorn/Projects/rust/wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot" # codegen-backends = ["cranelift"] +cc = "/home/bjorn/Projects/rust/wasi-sdk-24.0-x86_64-linux/bin/clang" +cxx = "/home/bjorn/Projects/rust/wasi-sdk-24.0-x86_64-linux/bin/clang" linker = "wasi-sdk-24.0-x86_64-linux/bin/clang" codegen-backends = ["llvm"] diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index ad7a7f3d03933..778939d8df282 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1253,7 +1253,9 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect // Building with a static libstdc++ is only supported on linux right now, // not for MSVC or macOS - if builder.config.llvm_static_stdcpp + if target.contains("wasi") { + cargo.env("LLVM_STATIC_STDCPP", dbg!(builder.wasi_libdir(target).unwrap().join("libc++.a"))); + } else if builder.config.llvm_static_stdcpp && !target.contains("freebsd") && !target.is_msvc() && !target.contains("apple") From b05a1af5d86b836c155086ecbb349950f71c97ac Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:14:26 +0200 Subject: [PATCH 06/12] Disable linker optimizations for wasm This is really slow when producing a 156MB rustc.wasm output. --- compiler/rustc_codegen_ssa/src/back/linker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index 34ec32e74d883..65e0d259c8c76 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -657,7 +657,7 @@ impl<'a> Linker for GccLinker<'a> { if self.sess.opts.optimize == config::OptLevel::Default || self.sess.opts.optimize == config::OptLevel::Aggressive { - self.link_arg("-O1"); + //self.link_arg("-O1"); } } From 094d63cd70f1cfd87b6a4766a0ad51d15b41e559 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 25 Sep 2024 12:55:20 +0200 Subject: [PATCH 07/12] Better linking of libc++ --- config.llvm.toml | 2 +- src/bootstrap/src/core/build_steps/compile.rs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config.llvm.toml b/config.llvm.toml index 7b5081f1a9075..290dd692dc711 100644 --- a/config.llvm.toml +++ b/config.llvm.toml @@ -33,7 +33,7 @@ prefix = "dist" sysconfdir = "etc" [target.'wasm32-wasip1-threads'] -wasi-root = "/home/bjorn/Projects/rust/wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot" +wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot" # codegen-backends = ["cranelift"] cc = "/home/bjorn/Projects/rust/wasi-sdk-24.0-x86_64-linux/bin/clang" cxx = "/home/bjorn/Projects/rust/wasi-sdk-24.0-x86_64-linux/bin/clang" diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 778939d8df282..a641546478f36 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1253,9 +1253,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect // Building with a static libstdc++ is only supported on linux right now, // not for MSVC or macOS - if target.contains("wasi") { - cargo.env("LLVM_STATIC_STDCPP", dbg!(builder.wasi_libdir(target).unwrap().join("libc++.a"))); - } else if builder.config.llvm_static_stdcpp + if builder.config.llvm_static_stdcpp && !target.contains("freebsd") && !target.is_msvc() && !target.contains("apple") @@ -1266,7 +1264,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect &builder.cxx(target).unwrap(), target, CLang::Cxx, - "libstdc++.a", + if target.contains("wasi") { "libc++.a" } else { "libstdc++.a" }, ); cargo.env("LLVM_STATIC_STDCPP", file); } From 51031c037ccaee92a74ad1ec76578f44488abaa7 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:34:35 +0200 Subject: [PATCH 08/12] Faster LLVM compiles --- config.llvm.toml | 2 ++ src/bootstrap/src/core/build_steps/llvm.rs | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/config.llvm.toml b/config.llvm.toml index 290dd692dc711..800b9c7c48e0e 100644 --- a/config.llvm.toml +++ b/config.llvm.toml @@ -17,6 +17,8 @@ ninja = false download-ci-llvm = false link-shared = false # thin-lto = true +targets = "WebAssembly;X86" +experimental-targets = "" [build] docs = false diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index af773f1e8ce3f..109251218096b 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -534,10 +534,6 @@ impl Step for Llvm { // Compiling C++ code requires a lot of stack space and can overflow and corrupt the heap. // (For example, `#include ` alone does it in a build with the default stack size.) let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -Wl,-z,stack-size=1048576 -Wl,--stack-first"); - // Some of the host APIs that are statically required by LLVM (notably threading) are dynamically - // never used. An LTO build removes imports of these APIs, simplifying deployment - let wasi_cflags_llvm = format!("{wasi_cflags_llvm} -flto"); - let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -flto -Wl,--strip-all"); // We need two toolchain files: one for the compiler itself (which needs threads at the moment since // -DLLVM_ENABLE_THREADS=OFF is kind of broken), and one for the runtime libs. From 34b540e6b1ce5a7cb642ac8fee954e8ef0127678 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:25:03 +0200 Subject: [PATCH 09/12] Remove some unnecessary configs --- config.llvm.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/config.llvm.toml b/config.llvm.toml index 800b9c7c48e0e..549aef07ca8dc 100644 --- a/config.llvm.toml +++ b/config.llvm.toml @@ -16,7 +16,6 @@ static-libstdcpp = true ninja = false download-ci-llvm = false link-shared = false -# thin-lto = true targets = "WebAssembly;X86" experimental-targets = "" @@ -24,10 +23,8 @@ experimental-targets = "" docs = false extended = false tools = [] -# host = ["wasm32-wasip1-threads", "x86_64-unknown-linux-gnu"] host = ["wasm32-wasip1-threads"] target = ["x86_64-unknown-linux-gnu", "wasm32-wasip1-threads"] -# target = ["wasm32-wasip1-threads"] cargo-native-static = true [install] @@ -36,9 +33,6 @@ sysconfdir = "etc" [target.'wasm32-wasip1-threads'] wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot" -# codegen-backends = ["cranelift"] -cc = "/home/bjorn/Projects/rust/wasi-sdk-24.0-x86_64-linux/bin/clang" -cxx = "/home/bjorn/Projects/rust/wasi-sdk-24.0-x86_64-linux/bin/clang" linker = "wasi-sdk-24.0-x86_64-linux/bin/clang" codegen-backends = ["llvm"] From b049100c85e804b26ee59e845d7c382b7784af04 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:36:56 +0200 Subject: [PATCH 10/12] Some preparation for once LLVM can compile for wasm32-wasip1 without threads --- config.llvm.toml | 7 ++++++- src/bootstrap/src/core/build_steps/llvm.rs | 14 +++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/config.llvm.toml b/config.llvm.toml index 549aef07ca8dc..d26dd81706abe 100644 --- a/config.llvm.toml +++ b/config.llvm.toml @@ -23,7 +23,7 @@ experimental-targets = "" docs = false extended = false tools = [] -host = ["wasm32-wasip1-threads"] +host = ["wasm32-wasip1"] target = ["x86_64-unknown-linux-gnu", "wasm32-wasip1-threads"] cargo-native-static = true @@ -31,6 +31,11 @@ cargo-native-static = true prefix = "dist" sysconfdir = "etc" +[target.'wasm32-wasip1'] +wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot" +linker = "wasi-sdk-24.0-x86_64-linux/bin/clang" +codegen-backends = ["llvm"] + [target.'wasm32-wasip1-threads'] wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot" linker = "wasi-sdk-24.0-x86_64-linux/bin/clang" diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 109251218096b..c3861a18d7c8d 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -524,7 +524,11 @@ impl Step for Llvm { let wasi_cflags = String::from(""); let wasi_ldflags = String::from(""); let wasi_target_llvm = target.triple.to_string(); - let wasi_cflags_llvm = format!("{wasi_cflags} -pthread"); + let wasi_cflags_llvm = if target.contains("threads") { + format!("{wasi_cflags} -pthread") + } else { + wasi_cflags.clone() + }; let wasi_ldflags_llvm = wasi_ldflags; // LLVM has some (unreachable in our configuration) calls to mmap. let wasi_cflags_llvm = format!("{wasi_cflags_llvm} -D_WASI_EMULATED_MMAN"); @@ -559,8 +563,6 @@ impl Step for Llvm { .define("LLVM_BUILD_SHARED_LIBS", "OFF") .define("LLVM_ENABLE_PIC", "OFF") .define("LLVM_BUILD_STATIC", "ON") - // .define("LLVM_ENABLE_THREADS", "OFF") - .define("LLVM_ENABLE_THREADS", "ON") .define("LLVM_BUILD_RUNTIME", "OFF") .define("LLVM_BUILD_TOOLS", "OFF") .define("LLVM_INCLUDE_UTILS", "OFF") @@ -678,6 +680,12 @@ impl Step for Llvm { .define("LLD_BUILD_TOOLS", "OFF") .define("CMAKE_BUILD_TYPE", "MinSizeRel") .define("HAVE_DLOPEN", ""); + + if target.contains("threads") { + cfg.define("LLVM_ENABLE_THREADS", "ON"); + } else { + cfg.define("LLVM_ENABLE_THREADS", "OFF"); + } } else { cfg.define("LLVM_TOOL_LLVM_CONFIG_BUILD", "ON") .define("LLVM_BUILD_TOOLS", "ON"); From a2b402d4e447667f3a3099d3344c6cda296d80e4 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:38:07 +0200 Subject: [PATCH 11/12] Rustfmt --- compiler/rustc_interface/src/util.rs | 6 +- compiler/rustc_llvm/build.rs | 1 - .../src/spec/targets/wasm32_wasip1_threads.rs | 8 +- src/bootstrap/src/core/build_steps/llvm.rs | 287 +++++++++--------- 4 files changed, 154 insertions(+), 148 deletions(-) diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 2c5e55127a07d..e199537e572be 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -103,10 +103,8 @@ fn run_in_thread_with_globals R + Send, R: Send>( // name contains null bytes. let r = builder .spawn_scoped(s, move || {*/ - rustc_span::create_session_globals_then(edition, Some(sm_inputs), || { - f(CurrentGcx::new()) - }) - /*}) + rustc_span::create_session_globals_then(edition, Some(sm_inputs), || f(CurrentGcx::new())) + /*}) .unwrap() .join(); diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index ff35568cfc8db..abc8f8bf974cc 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -205,7 +205,6 @@ fn main() { cfg.define("NDEBUG", None); } - if target.contains("wasi") { // ref src/bootstrap/src/core/build_steps/llvm.rs diff --git a/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs b/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs index 5b914f227bbe7..078ae9c6c9414 100644 --- a/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs +++ b/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs @@ -17,7 +17,13 @@ pub(crate) fn target() -> Target { options.add_pre_link_args( LinkerFlavor::WasmLld(Cc::No), - &["--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", "-lwasi-emulated-mman"], + &[ + "--import-memory", + "--export-memory", + "--shared-memory", + "--max-memory=1073741824", + "-lwasi-emulated-mman", + ], ); options.add_pre_link_args( LinkerFlavor::WasmLld(Cc::Yes), diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index c3861a18d7c8d..26444a2d1c8d1 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -517,7 +517,10 @@ impl Step for Llvm { if target.contains("wasi") { let wasi_sysroot = env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set"); - let wasi_sdk_path = std::path::Path::new(&wasi_sysroot).join("../../").canonicalize().expect("invalid WASI_SYSROOT"); + let wasi_sdk_path = std::path::Path::new(&wasi_sysroot) + .join("../../") + .canonicalize() + .expect("invalid WASI_SYSROOT"); let wasi_sysroot = format!("--sysroot={wasi_sysroot}"); let wasi_sdk_path = wasi_sdk_path.to_str().expect("invalid WASI_SYSROOT"); let wasi_target = target.triple.to_string(); @@ -537,149 +540,150 @@ impl Step for Llvm { let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -Wl,--max-memory=4294967296"); // Compiling C++ code requires a lot of stack space and can overflow and corrupt the heap. // (For example, `#include ` alone does it in a build with the default stack size.) - let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -Wl,-z,stack-size=1048576 -Wl,--stack-first"); + let wasi_ldflags_llvm = + format!("{wasi_ldflags_llvm} -Wl,-z,stack-size=1048576 -Wl,--stack-first"); // We need two toolchain files: one for the compiler itself (which needs threads at the moment since // -DLLVM_ENABLE_THREADS=OFF is kind of broken), and one for the runtime libs. cfg.define("WASI", "TRUE") - .define("CMAKE_SYSTEM_NAME", "Generic") - .define("CMAKE_SYSTEM_VERSION", "1") - .define("CMAKE_SYSTEM_PROCESSOR", "wasm32") - .define("CMAKE_EXECUTABLE_SUFFIX", ".wasm") - .define("CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER") - .define("CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", "ONLY") - .define("CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", "ONLY") - .define("CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", "ONLY") - .define("CMAKE_C_COMPILER", format!("{wasi_sdk_path}/bin/{wasi_target}-clang")) - .define("CMAKE_CXX_COMPILER", format!("{wasi_sdk_path}/bin/{wasi_target}-clang++")) - .define("CMAKE_LINKER", format!("{wasi_sdk_path}/bin/wasm-ld")) - .define("CMAKE_AR", format!("{wasi_sdk_path}/bin/ar")) - .define("CMAKE_RANLIB", format!("{wasi_sdk_path}/bin/ranlib")) - .define("CMAKE_C_COMPILER_TARGET", &wasi_target_llvm) - .define("CMAKE_CXX_COMPILER_TARGET", &wasi_target_llvm) - .define("CMAKE_C_FLAGS", format!("{wasi_sysroot} {wasi_cflags_llvm}")) - .define("CMAKE_CXX_FLAGS", format!("{wasi_sysroot} {wasi_cflags_llvm}")) - .define("CMAKE_EXE_LINKER_FLAGS", wasi_ldflags_llvm) - .define("LLVM_BUILD_SHARED_LIBS", "OFF") - .define("LLVM_ENABLE_PIC", "OFF") - .define("LLVM_BUILD_STATIC", "ON") - .define("LLVM_BUILD_RUNTIME", "OFF") - .define("LLVM_BUILD_TOOLS", "OFF") - .define("LLVM_INCLUDE_UTILS", "OFF") - .define("LLVM_BUILD_UTILS", "OFF") - .define("LLVM_INCLUDE_RUNTIMES", "OFF") - .define("LLVM_INCLUDE_EXAMPLES", "OFF") - .define("LLVM_INCLUDE_TESTS", "OFF") - .define("LLVM_INCLUDE_BENCHMARKS", "OFF") - .define("LLVM_INCLUDE_DOCS", "OFF") - .define("LLVM_TOOL_BUGPOINT_BUILD", "OFF") - .define("LLVM_TOOL_BUGPOINT_PASSES_BUILD", "OFF") - .define("LLVM_TOOL_DSYMUTIL_BUILD", "OFF") - .define("LLVM_TOOL_DXIL_DIS_BUILD", "OFF") - .define("LLVM_TOOL_GOLD_BUILD", "OFF") - .define("LLVM_TOOL_LLC_BUILD", "OFF") - .define("LLVM_TOOL_LLI_BUILD", "OFF") - // .define("LLVM_TOOL_LLVM_AR_BUILD", "ON") - // .define("LLVM_TOOL_LLVM_AS_BUILD", "ON") - .define("LLVM_TOOL_LLVM_AS_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_BCANALYZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_CAT_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_CFI_VERIFY_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_CONFIG_BUILD", "OFF") - // .define("LLVM_TOOL_LLVM_COV_BUILD", "ON") - .define("LLVM_TOOL_LLVM_CVTRES_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_CXXDUMP_BUILD", "OFF") - // .define("LLVM_TOOL_LLVM_CXXFILT_BUILD", "ON") - .define("LLVM_TOOL_LLVM_CXXMAP_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_C_TEST_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_DEBUGINFOD_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_DEBUGINFOD_FIND_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_DEBUGINFO_ANALYZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_DIFF_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_DIS_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_DIS_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_DLANG_DEMANGLE_FUZZER_BUILD", "OFF") - // .define("LLVM_TOOL_LLVM_DRIVER_BUILD", "ON") - // .define("LLVM_TOOL_LLVM_DWARFDUMP_BUILD", "ON") - .define("LLVM_TOOL_LLVM_DWARFUTIL_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_DWP_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_EXEGESIS_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_EXTRACT_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_GSYMUTIL_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_IFS_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_ISEL_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_ITANIUM_DEMANGLE_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_JITLINK_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_JITLISTENER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_LIBTOOL_DARWIN_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_LINK_BUILD", "ON") - .define("LLVM_TOOL_LLVM_LIPO_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_LTO2_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_LTO_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_MCA_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_MC_ASSEMBLE_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_MC_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_MC_DISASSEMBLE_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_MICROSOFT_DEMANGLE_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_ML_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_MODEXTRACT_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_MT_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_NM_BUILD", "OFF") - // .define("LLVM_TOOL_LLVM_OBJCOPY_BUILD", "ON") - // .define("LLVM_TOOL_LLVM_OBJDUMP_BUILD", "ON") - .define("LLVM_TOOL_LLVM_OPT_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_OPT_REPORT_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_PDBUTIL_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_PROFDATA_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_PROFGEN_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_RC_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_READOBJ_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_READTAPI_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_REDUCE_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_REMARKUTIL_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_RTDYLD_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_RUST_DEMANGLE_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_SHLIB_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_SIM_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_SIZE_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_SPLIT_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_STRESS_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_STRINGS_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_SYMBOLIZER_BUILD", "ON") - .define("LLVM_TOOL_LLVM_TLI_CHECKER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_UNDNAME_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_XRAY_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_YAML_NUMERIC_PARSER_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LLVM_YAML_PARSER_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_LTO_BUILD", "OFF") - .define("LLVM_TOOL_OBJ2YAML_BUILD", "OFF") - .define("LLVM_TOOL_OPT_BUILD", "OFF") - .define("LLVM_TOOL_OPT_VIEWER_BUILD", "OFF") - .define("LLVM_TOOL_REDUCE_CHUNK_LIST_BUILD", "OFF") - .define("LLVM_TOOL_REMARKS_SHLIB_BUILD", "OFF") - .define("LLVM_TOOL_SANCOV_BUILD", "OFF") - .define("LLVM_TOOL_SANSTATS_BUILD", "OFF") - .define("LLVM_TOOL_SPIRV_TOOLS_BUILD", "OFF") - .define("LLVM_TOOL_VERIFY_USELISTORDER_BUILD", "OFF") - .define("LLVM_TOOL_VFABI_DEMANGLE_FUZZER_BUILD", "OFF") - .define("LLVM_TOOL_XCODE_TOOLCHAIN_BUILD", "OFF") - .define("LLVM_TOOL_YAML2OBJ_BUILD", "OFF") - // .define("LLVM_ENABLE_PROJECTS", "clang;lld") - .define("LLVM_ENABLE_PROJECTS", "") - // .define("CLANG_ENABLE_ARCMT", "OFF") - // .define("CLANG_ENABLE_STATIC_ANALYZER", "OFF") - // .define("CLANG_INCLUDE_TESTS", "OFF") - // .define("CLANG_BUILD_TOOLS", "OFF") - // .define("CLANG_TOOL_CLANG_SCAN_DEPS_BUILD", "OFF") - // .define("CLANG_TOOL_CLANG_INSTALLAPI_BUILD", "OFF") - // .define("CLANG_BUILD_EXAMPLES", "OFF") - // .define("CLANG_INCLUDE_DOCS", "OFF") - // .define("CLANG_LINKS_TO_CREATE", "clang;clang++") - // .define("CLANG_LINKS_TO_CREATE", "") - .define("LLD_BUILD_TOOLS", "OFF") - .define("CMAKE_BUILD_TYPE", "MinSizeRel") - .define("HAVE_DLOPEN", ""); + .define("CMAKE_SYSTEM_NAME", "Generic") + .define("CMAKE_SYSTEM_VERSION", "1") + .define("CMAKE_SYSTEM_PROCESSOR", "wasm32") + .define("CMAKE_EXECUTABLE_SUFFIX", ".wasm") + .define("CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER") + .define("CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", "ONLY") + .define("CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", "ONLY") + .define("CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", "ONLY") + .define("CMAKE_C_COMPILER", format!("{wasi_sdk_path}/bin/{wasi_target}-clang")) + .define("CMAKE_CXX_COMPILER", format!("{wasi_sdk_path}/bin/{wasi_target}-clang++")) + .define("CMAKE_LINKER", format!("{wasi_sdk_path}/bin/wasm-ld")) + .define("CMAKE_AR", format!("{wasi_sdk_path}/bin/ar")) + .define("CMAKE_RANLIB", format!("{wasi_sdk_path}/bin/ranlib")) + .define("CMAKE_C_COMPILER_TARGET", &wasi_target_llvm) + .define("CMAKE_CXX_COMPILER_TARGET", &wasi_target_llvm) + .define("CMAKE_C_FLAGS", format!("{wasi_sysroot} {wasi_cflags_llvm}")) + .define("CMAKE_CXX_FLAGS", format!("{wasi_sysroot} {wasi_cflags_llvm}")) + .define("CMAKE_EXE_LINKER_FLAGS", wasi_ldflags_llvm) + .define("LLVM_BUILD_SHARED_LIBS", "OFF") + .define("LLVM_ENABLE_PIC", "OFF") + .define("LLVM_BUILD_STATIC", "ON") + .define("LLVM_BUILD_RUNTIME", "OFF") + .define("LLVM_BUILD_TOOLS", "OFF") + .define("LLVM_INCLUDE_UTILS", "OFF") + .define("LLVM_BUILD_UTILS", "OFF") + .define("LLVM_INCLUDE_RUNTIMES", "OFF") + .define("LLVM_INCLUDE_EXAMPLES", "OFF") + .define("LLVM_INCLUDE_TESTS", "OFF") + .define("LLVM_INCLUDE_BENCHMARKS", "OFF") + .define("LLVM_INCLUDE_DOCS", "OFF") + .define("LLVM_TOOL_BUGPOINT_BUILD", "OFF") + .define("LLVM_TOOL_BUGPOINT_PASSES_BUILD", "OFF") + .define("LLVM_TOOL_DSYMUTIL_BUILD", "OFF") + .define("LLVM_TOOL_DXIL_DIS_BUILD", "OFF") + .define("LLVM_TOOL_GOLD_BUILD", "OFF") + .define("LLVM_TOOL_LLC_BUILD", "OFF") + .define("LLVM_TOOL_LLI_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_AR_BUILD", "ON") + // .define("LLVM_TOOL_LLVM_AS_BUILD", "ON") + .define("LLVM_TOOL_LLVM_AS_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_BCANALYZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_CAT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_CFI_VERIFY_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_CONFIG_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_COV_BUILD", "ON") + .define("LLVM_TOOL_LLVM_CVTRES_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_CXXDUMP_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_CXXFILT_BUILD", "ON") + .define("LLVM_TOOL_LLVM_CXXMAP_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_C_TEST_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DEBUGINFOD_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DEBUGINFOD_FIND_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DEBUGINFO_ANALYZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DIFF_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DIS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DIS_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DLANG_DEMANGLE_FUZZER_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_DRIVER_BUILD", "ON") + // .define("LLVM_TOOL_LLVM_DWARFDUMP_BUILD", "ON") + .define("LLVM_TOOL_LLVM_DWARFUTIL_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_DWP_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_EXEGESIS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_EXTRACT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_GSYMUTIL_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_IFS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_ISEL_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_ITANIUM_DEMANGLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_JITLINK_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_JITLISTENER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_LIBTOOL_DARWIN_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_LINK_BUILD", "ON") + .define("LLVM_TOOL_LLVM_LIPO_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_LTO2_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_LTO_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MCA_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MC_ASSEMBLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MC_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MC_DISASSEMBLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MICROSOFT_DEMANGLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_ML_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MODEXTRACT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_MT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_NM_BUILD", "OFF") + // .define("LLVM_TOOL_LLVM_OBJCOPY_BUILD", "ON") + // .define("LLVM_TOOL_LLVM_OBJDUMP_BUILD", "ON") + .define("LLVM_TOOL_LLVM_OPT_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_OPT_REPORT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_PDBUTIL_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_PROFDATA_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_PROFGEN_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_RC_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_READOBJ_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_READTAPI_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_REDUCE_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_REMARKUTIL_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_RTDYLD_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_RUST_DEMANGLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SHLIB_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SIM_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SIZE_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SPLIT_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_STRESS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_STRINGS_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_SYMBOLIZER_BUILD", "ON") + .define("LLVM_TOOL_LLVM_TLI_CHECKER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_UNDNAME_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_XRAY_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_YAML_NUMERIC_PARSER_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LLVM_YAML_PARSER_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_LTO_BUILD", "OFF") + .define("LLVM_TOOL_OBJ2YAML_BUILD", "OFF") + .define("LLVM_TOOL_OPT_BUILD", "OFF") + .define("LLVM_TOOL_OPT_VIEWER_BUILD", "OFF") + .define("LLVM_TOOL_REDUCE_CHUNK_LIST_BUILD", "OFF") + .define("LLVM_TOOL_REMARKS_SHLIB_BUILD", "OFF") + .define("LLVM_TOOL_SANCOV_BUILD", "OFF") + .define("LLVM_TOOL_SANSTATS_BUILD", "OFF") + .define("LLVM_TOOL_SPIRV_TOOLS_BUILD", "OFF") + .define("LLVM_TOOL_VERIFY_USELISTORDER_BUILD", "OFF") + .define("LLVM_TOOL_VFABI_DEMANGLE_FUZZER_BUILD", "OFF") + .define("LLVM_TOOL_XCODE_TOOLCHAIN_BUILD", "OFF") + .define("LLVM_TOOL_YAML2OBJ_BUILD", "OFF") + // .define("LLVM_ENABLE_PROJECTS", "clang;lld") + .define("LLVM_ENABLE_PROJECTS", "") + // .define("CLANG_ENABLE_ARCMT", "OFF") + // .define("CLANG_ENABLE_STATIC_ANALYZER", "OFF") + // .define("CLANG_INCLUDE_TESTS", "OFF") + // .define("CLANG_BUILD_TOOLS", "OFF") + // .define("CLANG_TOOL_CLANG_SCAN_DEPS_BUILD", "OFF") + // .define("CLANG_TOOL_CLANG_INSTALLAPI_BUILD", "OFF") + // .define("CLANG_BUILD_EXAMPLES", "OFF") + // .define("CLANG_INCLUDE_DOCS", "OFF") + // .define("CLANG_LINKS_TO_CREATE", "clang;clang++") + // .define("CLANG_LINKS_TO_CREATE", "") + .define("LLD_BUILD_TOOLS", "OFF") + .define("CMAKE_BUILD_TYPE", "MinSizeRel") + .define("HAVE_DLOPEN", ""); if target.contains("threads") { cfg.define("LLVM_ENABLE_THREADS", "ON"); @@ -687,8 +691,7 @@ impl Step for Llvm { cfg.define("LLVM_ENABLE_THREADS", "OFF"); } } else { - cfg.define("LLVM_TOOL_LLVM_CONFIG_BUILD", "ON") - .define("LLVM_BUILD_TOOLS", "ON"); + cfg.define("LLVM_TOOL_LLVM_CONFIG_BUILD", "ON").define("LLVM_BUILD_TOOLS", "ON"); } cfg.build(); From a3d0b4fbeb5c6ab6a6bf44a2def3e024b9e05f1f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:07:58 +0200 Subject: [PATCH 12/12] Simplify LLVM build code a bit --- config.llvm.toml | 2 +- src/bootstrap/src/core/build_steps/llvm.rs | 25 +++++++++++----------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/config.llvm.toml b/config.llvm.toml index d26dd81706abe..1a64513e1ed48 100644 --- a/config.llvm.toml +++ b/config.llvm.toml @@ -23,7 +23,7 @@ experimental-targets = "" docs = false extended = false tools = [] -host = ["wasm32-wasip1"] +host = ["wasm32-wasip1-threads"] target = ["x86_64-unknown-linux-gnu", "wasm32-wasip1-threads"] cargo-native-static = true diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 26444a2d1c8d1..3c1acb2cf1f00 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -524,27 +524,26 @@ impl Step for Llvm { let wasi_sysroot = format!("--sysroot={wasi_sysroot}"); let wasi_sdk_path = wasi_sdk_path.to_str().expect("invalid WASI_SYSROOT"); let wasi_target = target.triple.to_string(); - let wasi_cflags = String::from(""); - let wasi_ldflags = String::from(""); let wasi_target_llvm = target.triple.to_string(); - let wasi_cflags_llvm = if target.contains("threads") { - format!("{wasi_cflags} -pthread") - } else { - wasi_cflags.clone() - }; - let wasi_ldflags_llvm = wasi_ldflags; + + let mut wasi_cflags_llvm = String::new(); + let mut wasi_ldflags_llvm = String::new(); + if target.contains("threads") { + wasi_cflags_llvm.push_str(" -pthread"); + } // LLVM has some (unreachable in our configuration) calls to mmap. - let wasi_cflags_llvm = format!("{wasi_cflags_llvm} -D_WASI_EMULATED_MMAN"); - let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -lwasi-emulated-mman"); + wasi_cflags_llvm.push_str(" -D_WASI_EMULATED_MMAN"); + wasi_ldflags_llvm.push_str(" -lwasi-emulated-mman"); // Depending on the code being compiled, both Clang and LLD can consume unbounded amounts of memory. - let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -Wl,--max-memory=4294967296"); + wasi_ldflags_llvm.push_str(" -Wl,--max-memory=4294967296"); // Compiling C++ code requires a lot of stack space and can overflow and corrupt the heap. // (For example, `#include ` alone does it in a build with the default stack size.) - let wasi_ldflags_llvm = - format!("{wasi_ldflags_llvm} -Wl,-z,stack-size=1048576 -Wl,--stack-first"); + wasi_ldflags_llvm.push_str(" -Wl,-z,stack-size=1048576 -Wl,--stack-first"); // We need two toolchain files: one for the compiler itself (which needs threads at the moment since // -DLLVM_ENABLE_THREADS=OFF is kind of broken), and one for the runtime libs. + // FIXME use wasi-sdk-24.0-x86_64-linux/share/cmake/wasi-sdk-pthread.cmake to define + // some of these variables in the future. cfg.define("WASI", "TRUE") .define("CMAKE_SYSTEM_NAME", "Generic") .define("CMAKE_SYSTEM_VERSION", "1")