From fa559fb2df28464a12c670c31ddd4068a4e53c88 Mon Sep 17 00:00:00 2001 From: Charles Lew Date: Mon, 26 Apr 2021 01:09:35 +0800 Subject: [PATCH 01/43] Implement RFC 1260 with feature_name `imported_main`. --- src/driver/jit.rs | 2 +- src/main_shim.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/driver/jit.rs b/src/driver/jit.rs index dbe1ff083f0db..f585ebca9dc40 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -103,7 +103,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { }); let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); - let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx); + let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); match entry_ty { EntryFnType::Main => { diff --git a/src/main_shim.rs b/src/main_shim.rs index a6266f507765f..dc86290b3fe93 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -13,7 +13,7 @@ pub(crate) fn maybe_create_entry_wrapper( ) { let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) { Some((def_id, entry_ty)) => ( - def_id.to_def_id(), + def_id, match entry_ty { EntryFnType::Main => true, EntryFnType::Start => false, From 95e6481d02ed600ca19775414231de774febe9a7 Mon Sep 17 00:00:00 2001 From: Alan Egerton Date: Fri, 30 Apr 2021 11:02:34 +0100 Subject: [PATCH 02/43] Set correct segment from #[link_section] for MachO --- src/constant.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/constant.rs b/src/constant.rs index 0a0e02d26394e..39daf887a22fe 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -374,8 +374,19 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant data_ctx.set_align(alloc.align.bytes()); if let Some(section_name) = section_name { - // FIXME set correct segment for Mach-O files - data_ctx.set_segment_section("", &*section_name); + let (segment_name, section_name) = if tcx.sess.target.is_like_osx { + if let Some(names) = section_name.split_once(',') { + names + } else { + tcx.sess.fatal(&format!( + "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma", + section_name + )); + } + } else { + ("", &*section_name) + }; + data_ctx.set_segment_section(segment_name, section_name); } let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec(); From ee570b1302b7c30cbd43047384622922eafc6c9c Mon Sep 17 00:00:00 2001 From: Erin Power Date: Fri, 30 Apr 2021 14:49:58 +0200 Subject: [PATCH 03/43] Sync rustc_codegen_cranelift 'ddd4ce25535cf71203ba3700896131ce55fde795' --- .github/workflows/main.yml | 7 + .vscode/settings.json | 2 +- Cargo.lock | 45 +- Cargo.toml | 6 +- Readme.md | 5 +- build_sysroot/Cargo.lock | 19 +- build_sysroot/prepare_sysroot_src.sh | 2 +- ...builtins-Remove-rotate_left-from-Int.patch | 4 +- docs/env_vars.md | 15 - example/mini_core_hello_world.rs | 43 +- example/std_example.rs | 3 + rust-toolchain | 2 +- scripts/ext_config.sh | 2 +- scripts/setup_rust_fork.sh | 2 +- scripts/test_rustc_tests.sh | 7 +- src/abi/mod.rs | 15 +- src/allocator.rs | 4 +- src/backend.rs | 20 +- src/base.rs | 146 ++---- src/common.rs | 20 +- src/config.rs | 107 ++++ src/constant.rs | 67 +-- src/debuginfo/unwind.rs | 66 +-- src/driver/aot.rs | 69 +-- src/driver/jit.rs | 153 +++--- src/driver/mod.rs | 58 +-- src/inline_asm.rs | 64 ++- src/intrinsics/cpuid.rs | 2 +- src/intrinsics/llvm.rs | 10 +- src/intrinsics/mod.rs | 144 +++--- src/intrinsics/simd.rs | 7 +- src/lib.rs | 136 ++--- src/linkage.rs | 1 + src/main_shim.rs | 70 ++- src/metadata.rs | 57 +- src/num.rs | 17 +- src/optimize/code_layout.rs | 34 -- src/optimize/mod.rs | 17 +- src/optimize/stack2reg.rs | 486 ------------------ src/pretty_clif.rs | 49 +- src/trap.rs | 6 +- src/value_and_place.rs | 2 +- src/vtable.rs | 36 +- 43 files changed, 784 insertions(+), 1243 deletions(-) delete mode 100644 docs/env_vars.md create mode 100644 src/config.rs delete mode 100644 src/optimize/code_layout.rs delete mode 100644 src/optimize/stack2reg.rs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ac516381cf7a..4d45e36c956c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -80,3 +80,10 @@ jobs: with: name: cg_clif-${{ runner.os }} path: cg_clif.tar.xz + + - name: Upload prebuilt cg_clif (cross compile) + if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + uses: actions/upload-artifact@v2 + with: + name: cg_clif-${{ runner.os }}-cross-x86_64-mingw + path: cg_clif.tar.xz diff --git a/.vscode/settings.json b/.vscode/settings.json index 0cd576e160f86..9009a532c54dc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this - "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"], + "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"], "rust-analyzer.assist.importMergeBehavior": "last", "rust-analyzer.cargo.runBuildScripts": true, "rust-analyzer.linkedProjects": [ diff --git a/Cargo.lock b/Cargo.lock index dc1cd336e1599..e6792def56796 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "byteorder", "cranelift-bforest", @@ -65,8 +65,8 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -74,18 +74,18 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" [[package]] name = "cranelift-entity" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" [[package]] name = "cranelift-frontend" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "cranelift-codegen", "log", @@ -95,8 +95,8 @@ dependencies = [ [[package]] name = "cranelift-jit" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "anyhow", "cranelift-codegen", @@ -113,8 +113,8 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "anyhow", "cranelift-codegen", @@ -125,8 +125,8 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -134,8 +134,8 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "anyhow", "cranelift-codegen", @@ -306,6 +306,7 @@ dependencies = [ "cranelift-frontend", "cranelift-jit", "cranelift-module", + "cranelift-native", "cranelift-object", "gimli", "indexmap", @@ -334,9 +335,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.11.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95" +checksum = "64ae3b39281e4b14b8123bdbaddd472b7dfe215e444181f2f9d2443c2444f834" [[package]] name = "thiserror" diff --git a/Cargo.toml b/Cargo.toml index 60946ab280858..2789207c65581 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,12 +9,13 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x64"] } +cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } +cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -target-lexicon = "0.11.0" +target-lexicon = "0.12.0" gimli = { version = "0.23.0", default-features = false, features = ["write"]} object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } @@ -28,6 +29,7 @@ smallvec = "1.6.1" #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } #cranelift-module = { path = "../wasmtime/cranelift/module" } +#cranelift-native = { path = ../wasmtime/cranelift/native" } #cranelift-jit = { path = "../wasmtime/cranelift/jit" } #cranelift-object = { path = "../wasmtime/cranelift/object" } diff --git a/Readme.md b/Readme.md index ffe1d9a1e6580..08f9373be6262 100644 --- a/Readme.md +++ b/Readme.md @@ -44,9 +44,10 @@ This will build your project with rustc_codegen_cranelift instead of the usual L For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md). -## Env vars +## Configuration -See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift. +See the documentation on the `BackendConfig` struct in [config.rs](src/config.rs) for all +configuration options. ## Not yet supported diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index 09c5d7590ab86..e058a972ead3c 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -56,7 +56,7 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.39" +version = "0.1.40" dependencies = [ "rustc-std-workspace-core", ] @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.91" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" dependencies = [ "rustc-std-workspace-core", ] @@ -167,6 +167,7 @@ dependencies = [ name = "panic_abort" version = "0.0.0" dependencies = [ + "alloc", "cfg-if", "compiler_builtins", "core", @@ -242,10 +243,22 @@ dependencies = [ "panic_abort", "panic_unwind", "rustc-demangle", + "std_detect", "unwind", "wasi", ] +[[package]] +name = "std_detect" +version = "0.1.5" +dependencies = [ + "cfg-if", + "compiler_builtins", + "libc", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + [[package]] name = "sysroot" version = "0.0.0" diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh index c90205db0fbd0..f7fcef1077410 100755 --- a/build_sysroot/prepare_sysroot_src.sh +++ b/build_sysroot/prepare_sysroot_src.sh @@ -32,7 +32,7 @@ popd git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned" pushd compiler-builtins git checkout -- . -git checkout 0.1.39 +git checkout 0.1.40 git apply ../../crate_patches/000*-compiler-builtins-*.patch popd diff --git a/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch b/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch index e14768910a9ac..b4acc4f5b7365 100644 --- a/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch +++ b/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch @@ -17,8 +17,8 @@ index 06054c8..3bea17b 100644 fn wrapping_shr(self, other: u32) -> Self; - fn rotate_left(self, other: u32) -> Self; fn overflowing_add(self, other: Self) -> (Self, bool); - fn aborting_div(self, other: Self) -> Self; - fn aborting_rem(self, other: Self) -> Self; + fn leading_zeros(self) -> u32; + } @@ -209,10 +208,6 @@ macro_rules! int_impl_common { ::wrapping_shr(self, other) } diff --git a/docs/env_vars.md b/docs/env_vars.md deleted file mode 100644 index f7fde1b4f3a87..0000000000000 --- a/docs/env_vars.md +++ /dev/null @@ -1,15 +0,0 @@ -# List of env vars recognized by cg_clif - -
-
CG_CLIF_JIT_ARGS
-
When JIT mode is enable pass these arguments to the program.
-
CG_CLIF_INCR_CACHE_DISABLED
-
Don't cache object files in the incremental cache. Useful during development of cg_clif - to make it possible to use incremental mode for all analyses performed by rustc without caching - object files when their content should have been changed by a change to cg_clif.
-
CG_CLIF_DISPLAY_CG_TIME
-
If "1", display the time it took to perform codegen for a crate.
-
CG_CLIF_ENABLE_VERIFIER
-
Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once - before passing the clif ir to Cranelift for compilation. -
diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index ea37ca98b59a7..6570f2bf9f297 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -11,6 +11,22 @@ unsafe extern "C" fn my_puts(s: *const i8) { puts(s); } +macro_rules! assert { + ($e:expr) => { + if !$e { + panic(stringify!(! $e)); + } + }; +} + +macro_rules! assert_eq { + ($l:expr, $r: expr) => { + if $l != $r { + panic(stringify!($l != $r)); + } + } +} + #[lang = "termination"] trait Termination { fn report(self) -> i32; @@ -20,8 +36,9 @@ impl Termination for () { fn report(self) -> i32 { unsafe { NUM = 6 * 7 + 1 + (1u8 == 1u8) as u8; // 44 - *NUM_REF as i32 + assert_eq!(*NUM_REF as i32, 44); } + 0 } } @@ -82,29 +99,12 @@ fn start( unsafe { puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); } } - main().report(); - 0 + main().report() as isize } static mut NUM: u8 = 6 * 7; static NUM_REF: &'static u8 = unsafe { &NUM }; -macro_rules! assert { - ($e:expr) => { - if !$e { - panic(stringify!(! $e)); - } - }; -} - -macro_rules! assert_eq { - ($l:expr, $r: expr) => { - if $l != $r { - panic(stringify!($l != $r)); - } - } -} - struct Unique { pointer: *const T, _marker: PhantomData, @@ -296,6 +296,11 @@ fn main() { unsafe { global_asm_test(); } + + // Both statics have a reference that points to the same anonymous allocation. + static REF1: &u8 = &42; + static REF2: &u8 = REF1; + assert_eq!(*REF1, *REF2); } #[cfg(all(not(jit), target_os = "linux"))] diff --git a/example/std_example.rs b/example/std_example.rs index 015bbdfed4648..77ba72df8ef37 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -48,6 +48,8 @@ fn main() { assert_eq!(2.3f32.copysign(-1.0), -2.3f32); println!("{}", 2.3f32.powf(2.0)); + assert_eq!(i64::MAX.checked_mul(2), None); + assert_eq!(-128i8, (-128i8).saturating_sub(1)); assert_eq!(127i8, 127i8.saturating_sub(-128)); assert_eq!(-128i8, (-128i8).saturating_add(-128)); @@ -84,6 +86,7 @@ fn main() { assert_eq!(houndred_i128 as f64, 100.0); assert_eq!(houndred_f32 as i128, 100); assert_eq!(houndred_f64 as i128, 100); + assert_eq!(1u128.rotate_left(2), 4); // Test signed 128bit comparing let max = usize::MAX as i128; diff --git a/rust-toolchain b/rust-toolchain index 2917fc7ee396d..5442e3345aa91 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-03-29" +channel = "nightly-2021-04-28" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/scripts/ext_config.sh b/scripts/ext_config.sh index 7971f620df14b..3f98d77d76cad 100644 --- a/scripts/ext_config.sh +++ b/scripts/ext_config.sh @@ -5,7 +5,7 @@ set -e export CG_CLIF_DISPLAY_CG_TIME=1 -export CG_CLIF_INCR_CACHE_DISABLED=1 +export CG_CLIF_DISABLE_INCR_CACHE=1 export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE} diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index e8bedf625f796..4821a07ac5d5d 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -45,7 +45,7 @@ index d95b5b7f17f..00b6f0e3635 100644 [dependencies] core = { path = "../core" } -compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] } -+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] } ++compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std', 'no-asm'] } [dev-dependencies] rand = "0.7" diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index fbc3feceec7ac..347fb40e6f9e7 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -38,6 +38,7 @@ rm src/test/ui/threads-sendsync/task-stderr.rs rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs rm src/test/ui/drop/drop-trait-enum.rs rm src/test/ui/numbers-arithmetic/issue-8460.rs +rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations rm src/test/ui/init-large-type.rs # same @@ -47,6 +48,7 @@ rm src/test/ui/issues/issue-51947.rs # same rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants rm src/test/ui/mir/mir_raw_fat_ptr.rs # same +rm src/test/ui/consts/issue-33537.rs # same rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same rm src/test/ui/generator/size-moved-locals.rs # same @@ -56,11 +58,13 @@ rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and n rm src/test/incremental/hashes/inline_asm.rs # inline asm rm src/test/incremental/issue-72386.rs # same -rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind rm src/test/incremental/issue-49482.rs # same rm src/test/incremental/issue-54059.rs # same rm src/test/incremental/lto.rs # requires lto +rm -r src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/ +rm -r src/test/run-make/unstable-flag-required # same + rm src/test/pretty/asm.rs # inline asm rm src/test/pretty/raw-str-nonexpr.rs # same @@ -68,6 +72,7 @@ rm -r src/test/run-pass-valgrind/unsized-locals rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning rm src/test/ui/json-bom-plus-crlf.rs # same +rm src/test/ui/match/issue-82392.rs # differing error rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 0e7829eaa26ac..54c8fb0e7b80b 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -63,16 +63,16 @@ pub(crate) fn import_function<'tcx>( module: &mut dyn Module, inst: Instance<'tcx>, ) -> FuncId { - let name = tcx.symbol_name(inst).name.to_string(); + let name = tcx.symbol_name(inst).name; let sig = get_function_sig(tcx, module.isa().triple(), inst); - module.declare_function(&name, Linkage::Import, &sig).unwrap() + module.declare_function(name, Linkage::Import, &sig).unwrap() } impl<'tcx> FunctionCx<'_, '_, 'tcx> { /// Instance must be monomorphized pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef { - let func_id = import_function(self.tcx, self.cx.module, inst); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = import_function(self.tcx, self.module, inst); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); if self.clif_comments.enabled() { self.add_comment(func_ref, format!("{:?}", inst)); @@ -89,8 +89,8 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { args: &[Value], ) -> &[Value] { let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) }; - let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap(); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); let call_inst = self.bcx.ins().call(func_ref, args); if self.clif_comments.enabled() { self.add_comment(call_inst, format!("easy_call {}", name)); @@ -295,7 +295,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ pub(crate) fn codegen_terminator_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, span: Span, - current_block: Block, func: &Operand<'tcx>, args: &[Operand<'tcx>], destination: Option<(Place<'tcx>, BasicBlock)>, @@ -357,7 +356,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD)) .unwrap_or(false); if is_cold { - fx.cold_blocks.insert(current_block); + // FIXME Mark current_block block as cold once Cranelift supports it } // Unpack arguments tuple for closures diff --git a/src/allocator.rs b/src/allocator.rs index f60645a9f97bc..a09e32577869e 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -11,7 +11,7 @@ use rustc_span::symbol::sym; pub(crate) fn codegen( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, ) -> bool { let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; @@ -29,7 +29,7 @@ pub(crate) fn codegen( fn codegen_inner( module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, kind: AllocatorKind, ) { let usize_ty = module.target_config().pointer_type(); diff --git a/src/backend.rs b/src/backend.rs index eb7927fc4adeb..05c06bac27db4 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -5,23 +5,23 @@ use std::convert::{TryFrom, TryInto}; use rustc_data_structures::fx::FxHashMap; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_module::FuncId; +use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; use object::write::*; use object::{RelocationEncoding, SectionKind, SymbolFlags}; -use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; - use gimli::SectionId; use crate::debuginfo::{DebugReloc, DebugRelocName}; pub(crate) trait WriteMetadata { - fn add_rustc_section(&mut self, symbol_name: String, data: Vec, is_like_osx: bool); + fn add_rustc_section(&mut self, symbol_name: String, data: Vec); } impl WriteMetadata for object::write::Object { - fn add_rustc_section(&mut self, symbol_name: String, data: Vec, _is_like_osx: bool) { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec) { let segment = self.segment_name(object::write::StandardSegment::Data).to_vec(); let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data); let offset = self.append_section_data(section_id, &data, 1); @@ -113,7 +113,7 @@ impl WriteDebugInfo for ObjectProduct { } pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec { - let triple = crate::build_isa(sess).triple().clone(); + let triple = crate::target_triple(sess); let binary_format = match triple.binary_format { target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, @@ -141,13 +141,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object metadata_object.write().unwrap() } -pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule { - let mut builder = ObjectBuilder::new( - crate::build_isa(sess), - name + ".o", - cranelift_module::default_libcall_names(), - ) - .unwrap(); +pub(crate) fn make_module(sess: &Session, isa: Box, name: String) -> ObjectModule { + let mut builder = + ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections // can easily double the amount of time necessary to perform linking. diff --git a/src/base.rs b/src/base.rs index b34a29c25b92e..3ec5c14ff17a2 100644 --- a/src/base.rs +++ b/src/base.rs @@ -6,9 +6,14 @@ use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiExt; use rustc_target::abi::call::FnAbi; +use crate::constant::ConstantCx; use crate::prelude::*; -pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { +pub(crate) fn codegen_fn<'tcx>( + cx: &mut crate::CodegenCx<'tcx>, + module: &mut dyn Module, + instance: Instance<'tcx>, +) { let tcx = cx.tcx; let _inst_guard = @@ -18,9 +23,9 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let mir = tcx.instance_mir(instance.def); // Declare function - let name = tcx.symbol_name(instance).name.to_string(); - let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); - let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap(); + let symbol_name = tcx.symbol_name(instance); + let sig = get_function_sig(tcx, module.isa().triple(), instance); + let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); cx.cached_context.clear(); @@ -39,15 +44,19 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect(); // Make FunctionCx - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); let mut fx = FunctionCx { cx, + module, tcx, pointer_type, + vtables: FxHashMap::default(), + constants_cx: ConstantCx::new(), instance, + symbol_name, mir, fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])), @@ -55,7 +64,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In block_map, local_map: IndexVec::with_capacity(mir.local_decls.len()), caller_location: None, // set by `codegen_fn_prelude` - cold_blocks: EntitySet::new(), clif_comments, source_info_set: indexmap::IndexSet::new(), @@ -90,7 +98,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let mut clif_comments = fx.clif_comments; let source_info_set = fx.source_info_set; let local_map = fx.local_map; - let cold_blocks = fx.cold_blocks; + + fx.constants_cx.finalize(fx.tcx, &mut *fx.module); // Store function in context let context = &mut cx.cached_context; @@ -103,21 +112,15 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In // Perform rust specific optimizations tcx.sess.time("optimize clif ir", || { - crate::optimize::optimize_function( - tcx, - instance, - context, - &cold_blocks, - &mut clif_comments, - ); + crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments); }); // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` // instruction, which doesn't have an encoding. context.compute_cfg(); context.compute_domtree(); - context.eliminate_unreachable_code(cx.module.isa()).unwrap(); - context.dce(cx.module.isa()).unwrap(); + context.eliminate_unreachable_code(module.isa()).unwrap(); + context.dce(module.isa()).unwrap(); // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't // invalidate it when it would change. context.domtree.clear(); @@ -125,7 +128,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In context.want_disasm = crate::pretty_clif::should_write_ir(tcx); // Define function - let module = &mut cx.module; tcx.sess.time("define function", || { module .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {}) @@ -136,7 +138,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In crate::pretty_clif::write_clif_file( tcx, "opt", - Some(cx.module.isa()), + Some(module.isa()), instance, &context, &clif_comments, @@ -145,13 +147,13 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { crate::pretty_clif::write_ir_file( tcx, - &format!("{}.vcode", tcx.symbol_name(instance).name), + || format!("{}.vcode", tcx.symbol_name(instance).name), |file| file.write_all(disasm.as_bytes()), ) } // Define debuginfo for function - let isa = cx.module.isa(); + let isa = module.isa(); let debug_context = &mut cx.debug_context; let unwind_context = &mut cx.unwind_context; tcx.sess.time("generate debug info", || { @@ -159,7 +161,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In debug_context.define_function( instance, func_id, - &name, + symbol_name.name, isa, context, &source_info_set, @@ -205,9 +207,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { // Unwinding after panicking is not supported continue; - // FIXME once unwinding is supported uncomment next lines - // // Unwinding is unlikely to happen, so mark cleanup block's as cold. - // fx.cold_blocks.insert(block); + // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do + // so for cleanup blocks. } fx.bcx.ins().nop(); @@ -262,7 +263,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { let target = fx.get_block(*target); let failure = fx.bcx.create_block(); - fx.cold_blocks.insert(failure); + // FIXME Mark failure block as cold once Cranelift supports it if *expected { fx.bcx.ins().brz(cond, failure, &[]); @@ -355,14 +356,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { from_hir_call: _, } => { fx.tcx.sess.time("codegen call", || { - crate::abi::codegen_terminator_call( - fx, - *fn_span, - block, - func, - args, - *destination, - ) + crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination) }); } TerminatorKind::InlineAsm { @@ -664,7 +658,7 @@ fn codegen_stmt<'tcx>( // FIXME use emit_small_memset where possible let addr = lval.to_ptr().get_addr(fx); let val = operand.load_scalar(fx); - fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times); + fx.bcx.call_memset(fx.module.target_config(), addr, val, times); } else { let loop_block = fx.bcx.create_block(); let loop_block2 = fx.bcx.create_block(); @@ -750,85 +744,15 @@ fn codegen_stmt<'tcx>( | StatementKind::AscribeUserType(..) => {} StatementKind::LlvmInlineAsm(asm) => { - use rustc_span::symbol::Symbol; - let LlvmInlineAsm { asm, outputs, inputs } = &**asm; - let rustc_hir::LlvmInlineAsmInner { - asm: asm_code, // Name - outputs: output_names, // Vec - inputs: input_names, // Vec - clobbers, // Vec - volatile, // bool - alignstack, // bool - dialect: _, - asm_str_style: _, - } = asm; - match asm_code.as_str().trim() { + match asm.asm.asm.as_str().trim() { "" => { // Black box } - "mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => { - assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]); - assert_eq!(output_names.len(), 4); - for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() { - assert_eq!(&output_names[i].constraint.as_str(), c); - assert!(!output_names[i].is_rw); - assert!(!output_names[i].is_indirect); - } - - assert_eq!(clobbers, &[]); - - assert!(!volatile); - assert!(!alignstack); - - assert_eq!(inputs.len(), 2); - let leaf = codegen_operand(fx, &inputs[0].1).load_scalar(fx); // %eax - let subleaf = codegen_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx - - let (eax, ebx, ecx, edx) = - crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf); - - assert_eq!(outputs.len(), 4); - codegen_place(fx, outputs[0]) - .write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[1]) - .write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[2]) - .write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[3]) - .write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); - } - "xgetbv" => { - assert_eq!(input_names, &[Symbol::intern("{ecx}")]); - - assert_eq!(output_names.len(), 2); - for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() { - assert_eq!(&output_names[i].constraint.as_str(), c); - assert!(!output_names[i].is_rw); - assert!(!output_names[i].is_indirect); - } - - assert_eq!(clobbers, &[]); - - assert!(!volatile); - assert!(!alignstack); - - crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported"); - } - // ___chkstk, ___chkstk_ms and __alloca are only used on Windows - _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => { - crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); - } - _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => { - crate::trap::trap_unimplemented(fx, "Alloca is not supported"); - } - // Used in sys::windows::abort_internal - "int $$0x29" => { - crate::trap::trap_unimplemented(fx, "Windows abort"); - } - _ => fx - .tcx - .sess - .span_fatal(stmt.source_info.span, "Inline assembly is not supported"), + _ => fx.tcx.sess.span_fatal( + stmt.source_info.span, + "Legacy `llvm_asm!` inline assembly is not supported. \ + Try using the new `asm!` instead.", + ), } } StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), @@ -844,7 +768,7 @@ fn codegen_stmt<'tcx>( let elem_size: u64 = pointee.size.bytes(); let bytes = if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes); } } } diff --git a/src/common.rs b/src/common.rs index b5874f62535ca..92e4435565ee7 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,8 +1,10 @@ use rustc_index::vec::IndexVec; +use rustc_middle::ty::SymbolName; use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Integer, Primitive}; use rustc_target::spec::{HasTargetSpec, Target}; +use crate::constant::ConstantCx; use crate::prelude::*; pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type { @@ -226,12 +228,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { } } -pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { - pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>, +pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { + pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>, + pub(crate) module: &'m mut dyn Module, pub(crate) tcx: TyCtxt<'tcx>, pub(crate) pointer_type: Type, // Cached from module + pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, + pub(crate) constants_cx: ConstantCx, pub(crate) instance: Instance<'tcx>, + pub(crate) symbol_name: SymbolName<'tcx>, pub(crate) mir: &'tcx Body<'tcx>, pub(crate) fn_abi: Option>>, @@ -242,9 +248,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { /// When `#[track_caller]` is used, the implicit caller location is stored in this variable. pub(crate) caller_location: Option>, - /// See [`crate::optimize::code_layout`] for more information. - pub(crate) cold_blocks: EntitySet, - pub(crate) clif_comments: crate::pretty_clif::CommentWriter, pub(crate) source_info_set: indexmap::IndexSet, @@ -339,7 +342,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { } pub(crate) fn triple(&self) -> &target_lexicon::Triple { - self.cx.module.isa().triple() + self.module.isa().triple() } pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value { @@ -352,15 +355,14 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let mut data_ctx = DataContext::new(); data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice()); let msg_id = self - .cx .module .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false) .unwrap(); // Ignore DuplicateDefinition error, as the data will be the same - let _ = self.cx.module.define_data(msg_id, &data_ctx); + let _ = self.module.define_data(msg_id, &data_ctx); - let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func); + let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func); if self.clif_comments.enabled() { self.add_comment(local_msg_id, msg); } diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000000000..e59a0cb0a2323 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,107 @@ +use std::env; +use std::str::FromStr; + +fn bool_env_var(key: &str) -> bool { + env::var(key).as_ref().map(|val| &**val) == Ok("1") +} + +/// The mode to use for compilation. +#[derive(Copy, Clone, Debug)] +pub enum CodegenMode { + /// AOT compile the crate. This is the default. + Aot, + /// JIT compile and execute the crate. + Jit, + /// JIT compile and execute the crate, but only compile functions the first time they are used. + JitLazy, +} + +impl FromStr for CodegenMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "aot" => Ok(CodegenMode::Aot), + "jit" => Ok(CodegenMode::Jit), + "jit-lazy" => Ok(CodegenMode::JitLazy), + _ => Err(format!("Unknown codegen mode `{}`", s)), + } + } +} + +/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars. +#[derive(Clone, Debug)] +pub struct BackendConfig { + /// Should the crate be AOT compiled or JIT executed. + /// + /// Defaults to AOT compilation. Can be set using `-Cllvm-args=mode=...`. + pub codegen_mode: CodegenMode, + + /// When JIT mode is enable pass these arguments to the program. + /// + /// Defaults to the value of `CG_CLIF_JIT_ARGS`. + pub jit_args: Vec, + + /// Display the time it took to perform codegen for a crate. + /// + /// Defaults to true when the `CG_CLIF_DISPLAY_CG_TIME` env var is set to 1 or false otherwise. + /// Can be set using `-Cllvm-args=display_cg_time=...`. + pub display_cg_time: bool, + + /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run + /// once before passing the clif ir to Cranelift for compilation. + /// + /// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is + /// compiled with debug assertions enabled or false otherwise. Can be set using + /// `-Cllvm-args=enable_verifier=...`. + pub enable_verifier: bool, + + /// Don't cache object files in the incremental cache. Useful during development of cg_clif + /// to make it possible to use incremental mode for all analyses performed by rustc without + /// caching object files when their content should have been changed by a change to cg_clif. + /// + /// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false + /// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`. + pub disable_incr_cache: bool, +} + +impl Default for BackendConfig { + fn default() -> Self { + BackendConfig { + codegen_mode: CodegenMode::Aot, + jit_args: { + let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); + args.split(' ').map(|arg| arg.to_string()).collect() + }, + display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"), + enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"), + disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"), + } + } +} + +impl BackendConfig { + /// Parse the configuration passed in using `-Cllvm-args`. + pub fn from_opts(opts: &[String]) -> Result { + fn parse_bool(name: &str, value: &str) -> Result { + value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name)) + } + + let mut config = BackendConfig::default(); + for opt in opts { + if let Some((name, value)) = opt.split_once('=') { + match name { + "mode" => config.codegen_mode = value.parse()?, + "display_cg_time" => config.display_cg_time = parse_bool(name, value)?, + "enable_verifier" => config.enable_verifier = parse_bool(name, value)?, + "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?, + _ => return Err(format!("Unknown option `{}`", name)), + } + } else { + return Err(format!("Invalid option `{}`", opt)); + } + } + + Ok(config) + } +} diff --git a/src/constant.rs b/src/constant.rs index fcd41c844659d..0a0e02d26394e 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -2,7 +2,7 @@ use rustc_span::DUMMY_SP; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::ErrorReported; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::interpret::{ @@ -15,10 +15,10 @@ use cranelift_module::*; use crate::prelude::*; -#[derive(Default)] pub(crate) struct ConstantCx { todo: Vec, done: FxHashSet, + anon_allocs: FxHashMap, } #[derive(Copy, Clone, Debug)] @@ -28,6 +28,10 @@ enum TodoItem { } impl ConstantCx { + pub(crate) fn new() -> Self { + ConstantCx { todo: vec![], done: FxHashSet::default(), anon_allocs: FxHashMap::default() } + } + pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) { //println!("todo {:?}", self.todo); define_all_allocs(tcx, module, &mut self); @@ -74,8 +78,10 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { all_constants_ok } -pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) { +pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) { + let mut constants_cx = ConstantCx::new(); constants_cx.todo.push(TodoItem::Static(def_id)); + constants_cx.finalize(tcx, module); } pub(crate) fn codegen_tls_ref<'tcx>( @@ -83,8 +89,8 @@ pub(crate) fn codegen_tls_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("tls {:?}", def_id)); } @@ -97,8 +103,8 @@ fn codegen_static_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CPlace<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -182,28 +188,31 @@ pub(crate) fn codegen_const_value<'tcx>( let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id); let base_addr = match alloc_kind { Some(GlobalAlloc::Memory(alloc)) => { - fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); - let data_id = - data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability); + fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); + let data_id = data_id_for_alloc_id( + &mut fx.constants_cx, + fx.module, + ptr.alloc_id, + alloc.mutability, + ); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); } fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } Some(GlobalAlloc::Function(instance)) => { - let func_id = - crate::abi::import_function(fx.tcx, fx.cx.module, instance); + let func_id = crate::abi::import_function(fx.tcx, fx.module, instance); let local_func_id = - fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + fx.module.declare_func_in_func(func_id, &mut fx.bcx.func); fx.bcx.ins().func_addr(fx.pointer_type, local_func_id) } Some(GlobalAlloc::Static(def_id)) => { assert!(fx.tcx.is_static(def_id)); - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -243,10 +252,11 @@ fn pointer_for_allocation<'tcx>( alloc: &'tcx Allocation, ) -> crate::pointer::Pointer { let alloc_id = fx.tcx.create_memory_alloc(alloc); - fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); - let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability); + fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); + let data_id = + data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", alloc_id)); } @@ -255,18 +265,14 @@ fn pointer_for_allocation<'tcx>( } fn data_id_for_alloc_id( + cx: &mut ConstantCx, module: &mut dyn Module, alloc_id: AllocId, mutability: rustc_hir::Mutability, ) -> DataId { - module - .declare_data( - &format!(".L__alloc_{:x}", alloc_id.0), - Linkage::Local, - mutability == rustc_hir::Mutability::Mut, - false, - ) - .unwrap() + *cx.anon_allocs.entry(alloc_id).or_insert_with(|| { + module.declare_anonymous_data(mutability == rustc_hir::Mutability::Mut, false).unwrap() + }) } fn data_id_for_static( @@ -344,7 +350,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant GlobalAlloc::Memory(alloc) => alloc, GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(), }; - let data_id = data_id_for_alloc_id(module, alloc_id, alloc.mutability); + let data_id = data_id_for_alloc_id(cx, module, alloc_id, alloc.mutability); (data_id, alloc, None) } TodoItem::Static(def_id) => { @@ -397,7 +403,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant } GlobalAlloc::Memory(target_alloc) => { cx.todo.push(TodoItem::Alloc(reloc)); - data_id_for_alloc_id(module, reloc, target_alloc.mutability) + data_id_for_alloc_id(cx, module, reloc, target_alloc.mutability) } GlobalAlloc::Static(def_id) => { if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) @@ -419,8 +425,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64); } - // FIXME don't duplicate definitions in lazy jit mode - let _ = module.define_data(data_id, &data_ctx); + module.define_data(data_id, &data_ctx).unwrap(); cx.done.insert(data_id); } diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index 357c9fe6ed83a..ca7083cccb88f 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -5,17 +5,19 @@ use crate::prelude::*; use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; use gimli::write::{Address, CieId, EhFrame, FrameTable, Section}; +use gimli::RunTimeEndian; use crate::backend::WriteDebugInfo; -pub(crate) struct UnwindContext<'tcx> { - tcx: TyCtxt<'tcx>, +pub(crate) struct UnwindContext { + endian: RunTimeEndian, frame_table: FrameTable, cie_id: Option, } -impl<'tcx> UnwindContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { +impl UnwindContext { + pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { + let endian = super::target_endian(tcx); let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { @@ -28,7 +30,7 @@ impl<'tcx> UnwindContext<'tcx> { None }; - UnwindContext { tcx, frame_table, cie_id } + UnwindContext { endian, frame_table, cie_id } } pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) { @@ -54,8 +56,7 @@ impl<'tcx> UnwindContext<'tcx> { } pub(crate) fn emit(self, product: &mut P) { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if !eh_frame.0.writer.slice().is_empty() { @@ -71,16 +72,12 @@ impl<'tcx> UnwindContext<'tcx> { } #[cfg(feature = "jit")] - pub(crate) unsafe fn register_jit( - self, - jit_module: &cranelift_jit::JITModule, - ) -> Option { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) { + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if eh_frame.0.writer.slice().is_empty() { - return None; + return; } let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module); @@ -88,7 +85,10 @@ impl<'tcx> UnwindContext<'tcx> { // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. eh_frame.extend(&[0, 0, 0, 0]); - let mut registrations = Vec::new(); + // FIXME support unregistering unwind tables once cranelift-jit supports deallocating + // individual functions + #[allow(unused_variables)] + let (eh_frame, eh_frame_len, _) = Vec::into_raw_parts(eh_frame); // ======================================================================= // Everything after this line up to the end of the file is loosly based on @@ -96,8 +96,8 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(target_os = "macos")] { // On macOS, `__register_frame` takes a pointer to a single FDE - let start = eh_frame.as_ptr(); - let end = start.add(eh_frame.len()); + let start = eh_frame; + let end = start.add(eh_frame_len); let mut current = start; // Walk all of the entries in the frame table and register them @@ -107,7 +107,6 @@ impl<'tcx> UnwindContext<'tcx> { // Skip over the CIE if current != start { __register_frame(current); - registrations.push(current as usize); } // Move to the next table entry (+4 because the length itself is not inclusive) @@ -117,41 +116,12 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(not(target_os = "macos"))] { // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0 - let ptr = eh_frame.as_ptr(); - __register_frame(ptr); - registrations.push(ptr as usize); + __register_frame(eh_frame); } - - Some(UnwindRegistry { _frame_table: eh_frame, registrations }) } } -/// Represents a registry of function unwind information for System V ABI. -pub(crate) struct UnwindRegistry { - _frame_table: Vec, - registrations: Vec, -} - extern "C" { // libunwind import fn __register_frame(fde: *const u8); - fn __deregister_frame(fde: *const u8); -} - -impl Drop for UnwindRegistry { - fn drop(&mut self) { - unsafe { - // libgcc stores the frame entries as a linked list in decreasing sort order - // based on the PC value of the registered entry. - // - // As we store the registrations in increasing order, it would be O(N^2) to - // deregister in that order. - // - // To ensure that we just pop off the first element in the list upon every - // deregistration, walk our list of registrations backwards. - for fde in self.registrations.iter().rev() { - __deregister_frame(*fde as *const _); - } - } - } } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index ed3bdedddced5..9c5cd53d8669d 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -16,12 +16,6 @@ use cranelift_object::ObjectModule; use crate::{prelude::*, BackendConfig}; -fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule { - let module = crate::backend::make_module(tcx.sess, name); - assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); - module -} - struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); impl HashStable for ModuleCodegenResult { @@ -32,11 +26,12 @@ impl HashStable for ModuleCodegenResult { fn emit_module( tcx: TyCtxt<'_>, + backend_config: &BackendConfig, name: String, kind: ModuleKind, module: ObjectModule, debug: Option>, - unwind_context: UnwindContext<'_>, + unwind_context: UnwindContext, ) -> ModuleCodegenResult { let mut product = module.finish(); @@ -52,7 +47,7 @@ fn emit_module( tcx.sess.fatal(&format!("error writing object file: {}", err)); } - let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() { + let work_product = if backend_config.disable_incr_cache { None } else { rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( @@ -110,23 +105,24 @@ fn module_codegen( let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); - let mut module = new_module(tcx, cgu_name.as_str().to_string()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string()); let mut cx = crate::CodegenCx::new( tcx, - backend_config, - &mut module, + backend_config.clone(), + module.isa(), tcx.sess.opts.debuginfo != DebugInfo::None, ); - super::predefine_mono_items(&mut cx, &mono_items); + super::predefine_mono_items(tcx, &mut module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); - } - MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id) + cx.tcx + .sess + .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst)); } + MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { let item = cx.tcx.hir().item(item_id); if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { @@ -138,25 +134,28 @@ fn module_codegen( } } } - let (global_asm, debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context, false); - let codegen_result = emit_module( - tcx, - cgu.name().as_str().to_string(), - ModuleKind::Regular, - module, - debug, - unwind_context, - ); + let debug_context = cx.debug_context; + let unwind_context = cx.unwind_context; + let codegen_result = tcx.sess.time("write object file", || { + emit_module( + tcx, + &backend_config, + cgu.name().as_str().to_string(), + ModuleKind::Regular, + module, + debug_context, + unwind_context, + ) + }); - codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm); + codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm); codegen_result } -pub(super) fn run_aot( +pub(crate) fn run_aot( tcx: TyCtxt<'_>, backend_config: BackendConfig, metadata: EncodedMetadata, @@ -193,14 +192,14 @@ pub(super) fn run_aot( } } - let modules = super::time(tcx, "codegen mono items", || { + let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { cgus.iter() .map(|cgu| { let cgu_reuse = determine_cgu_reuse(tcx, cgu); tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse); match cgu_reuse { - _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {} + _ if backend_config.disable_incr_cache => {} CguReuse::No => {} CguReuse::PreLto => { return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products); @@ -212,7 +211,7 @@ pub(super) fn run_aot( let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task( dep_node, tcx, - (backend_config, cgu.name()), + (backend_config.clone(), cgu.name()), module_codegen, rustc_middle::dep_graph::hash_result, ); @@ -228,7 +227,10 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); - let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut allocator_module = + crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string()); + assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); @@ -236,6 +238,7 @@ pub(super) fn run_aot( let allocator_module = if created_alloc_shim { let ModuleCodegenResult(module, work_product) = emit_module( tcx, + &backend_config, "allocator_shim".to_string(), ModuleKind::Allocator, allocator_module, diff --git a/src/driver/jit.rs b/src/driver/jit.rs index f585ebca9dc40..956207c5b7cfe 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -1,4 +1,4 @@ -//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object +//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object //! files. use std::cell::RefCell; @@ -8,32 +8,62 @@ use std::os::raw::{c_char, c_int}; use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; -use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; use crate::{prelude::*, BackendConfig}; use crate::{CodegenCx, CodegenMode}; -thread_local! { - pub static BACKEND_CONFIG: RefCell> = RefCell::new(None); - pub static CURRENT_MODULE: RefCell> = RefCell::new(None); +struct JitState { + backend_config: BackendConfig, + jit_module: JITModule, } -pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { - if !tcx.sess.opts.output_types.should_codegen() { - tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); - } +thread_local! { + static LAZY_JIT_STATE: RefCell> = RefCell::new(None); +} +fn create_jit_module<'tcx>( + tcx: TyCtxt<'tcx>, + backend_config: &BackendConfig, + hotswap: bool, +) -> (JITModule, CodegenCx<'tcx>) { let imported_symbols = load_imported_symbols_for_jit(tcx); - let mut jit_builder = - JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names()); - jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); + let isa = crate::build_isa(tcx.sess, backend_config); + let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); + jit_builder.hotswap(hotswap); crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); jit_builder.symbols(imported_symbols); let mut jit_module = JITModule::new(jit_builder); - assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); + + let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false); + + crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); + crate::main_shim::maybe_create_entry_wrapper( + tcx, + &mut jit_module, + &mut cx.unwind_context, + true, + ); + + (jit_module, cx) +} + +pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { + if !tcx.sess.opts.output_types.should_codegen() { + tcx.sess.fatal("JIT mode doesn't work with `cargo check`"); + } + + if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) { + tcx.sess.fatal("can't jit non-executable crate"); + } + + let (mut jit_module, mut cx) = create_jit_module( + tcx, + &backend_config, + matches!(backend_config.codegen_mode, CodegenMode::JitLazy), + ); let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); let mono_items = cgus @@ -44,52 +74,45 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .into_iter() .collect::>(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false); - - super::time(tcx, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); + super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { + super::predefine_mono_items(tcx, &mut jit_module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => match backend_config.codegen_mode { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, &mut jit_module, inst) + }); } - CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), }, MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id); + crate::constant::codegen_static(tcx, &mut jit_module, def_id); } MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx.hir().item(item_id); + let item = tcx.hir().item(item_id); tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode"); } } } }); - let (global_asm, _debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - jit_module.finalize_definitions(); - - if !global_asm.is_empty() { + if !cx.global_asm.is_empty() { tcx.sess.fatal("Inline asm is not supported in JIT mode"); } - crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); - tcx.sess.abort_if_errors(); jit_module.finalize_definitions(); - let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; + unsafe { cx.unwind_context.register_jit(&jit_module) }; println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" ); - let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) - .chain(args.split(' ')) + .chain(backend_config.jit_args.iter().map(|arg| &**arg)) .map(|arg| CString::new(arg).unwrap()) .collect::>(); let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::>(); @@ -98,8 +121,21 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); - BACKEND_CONFIG.with(|tls_backend_config| { - assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) + let start_sig = Signature { + params: vec![ + AbiParam::new(jit_module.target_config().pointer_type()), + AbiParam::new(jit_module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], + call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), + }; + let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); + let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); + + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + assert!(lazy_jit_state.is_none()); + *lazy_jit_state = Some(JitState { backend_config, jit_module }); }); let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); @@ -161,24 +197,23 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 // lift is used to ensure the correct lifetime for instance. let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); - CURRENT_MODULE.with(|jit_module| { - let mut jit_module = jit_module.borrow_mut(); - let jit_module = jit_module.as_mut().unwrap(); - let backend_config = - BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap()); + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); + let jit_module = &mut lazy_jit_state.jit_module; + let backend_config = lazy_jit_state.backend_config.clone(); - let name = tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name; let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance); - let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap(); jit_module.prepare_for_function_redefine(func_id).unwrap(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false); - tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance)); + let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false); + tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance)); - let (global_asm, _debug_context, unwind_context) = cx.finalize(); - assert!(global_asm.is_empty()); + assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); - std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + unsafe { cx.unwind_context.register_jit(&jit_module) }; jit_module.get_finalized_function(func_id) }) }) @@ -248,35 +283,37 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } -fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { +fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) { let tcx = cx.tcx; - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); - let name = tcx.symbol_name(inst).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); - let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let name = tcx.symbol_name(inst).name; + let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst); + let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap(); let instance_ptr = Box::into_raw(Box::new(inst)); - let jit_fn = cx - .module + let jit_fn = module .declare_function( "__clif_jit_fn", Linkage::Import, &Signature { - call_conv: cx.module.target_config().default_call_conv, + call_conv: module.target_config().default_call_conv, params: vec![AbiParam::new(pointer_type)], returns: vec![AbiParam::new(pointer_type)], }, ) .unwrap(); - let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + cx.cached_context.clear(); + let trampoline = &mut cx.cached_context.func; + trampoline.signature = sig.clone(); + let mut builder_ctx = FunctionBuilderContext::new(); - let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); - let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func); + let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func); let sig_ref = trampoline_builder.func.import_signature(sig); let entry_block = trampoline_builder.create_block(); @@ -291,10 +328,10 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); trampoline_builder.ins().return_(&ret_vals); - cx.module + module .define_function( func_id, - &mut Context::for_function(trampoline), + &mut cx.cached_context, &mut NullTrapSink {}, &mut NullStackMapSink {}, ) diff --git a/src/driver/mod.rs b/src/driver/mod.rs index d49182a07b79e..8f5714ecb4177 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -1,63 +1,37 @@ -//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions -//! like JIT executing or writing object files. +//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and +//! performing any further actions like JIT executing or writing object files. +//! +//! [`codegen_fn`]: crate::base::codegen_fn +//! [`codegen_static`]: crate::constant::codegen_static -use std::any::Any; - -use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; -use crate::CodegenMode; -mod aot; +pub(crate) mod aot; #[cfg(feature = "jit")] -mod jit; - -pub(crate) fn codegen_crate( - tcx: TyCtxt<'_>, - metadata: EncodedMetadata, - need_metadata_module: bool, - backend_config: crate::BackendConfig, -) -> Box { - tcx.sess.abort_if_errors(); - - match backend_config.codegen_mode { - CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module), - CodegenMode::Jit | CodegenMode::JitLazy => { - let is_executable = - tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } - - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx, backend_config); - - #[cfg(not(feature = "jit"))] - tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); - } - } -} +pub(crate) mod jit; fn predefine_mono_items<'tcx>( - cx: &mut crate::CodegenCx<'_, 'tcx>, + tcx: TyCtxt<'tcx>, + module: &mut dyn Module, mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))], ) { - cx.tcx.sess.time("predefine functions", || { - let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE); + tcx.sess.time("predefine functions", || { + let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE); for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { - let name = cx.tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name; let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); - let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); + let sig = get_function_sig(tcx, module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage( mono_item, linkage, visibility, is_compiler_builtins, ); - cx.module.declare_function(&name, linkage, &sig).unwrap(); + module.declare_function(name, linkage, &sig).unwrap(); } MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} } @@ -65,8 +39,8 @@ fn predefine_mono_items<'tcx>( }); } -fn time(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { - if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") { +fn time(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R { + if display { println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name); let before = std::time::Instant::now(); let res = tcx.sess.time(name, f); diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 1fb5e86aed7df..4ab4c2957ca4e 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -24,6 +24,64 @@ pub(crate) fn codegen_inline_asm<'tcx>( let true_ = fx.bcx.ins().iconst(types::I32, 1); fx.bcx.ins().trapnz(true_, TrapCode::User(1)); return; + } else if template[0] == InlineAsmTemplatePiece::String("mov rsi, rbx".to_string()) + && template[1] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[2] == InlineAsmTemplatePiece::String("cpuid".to_string()) + && template[3] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[4] == InlineAsmTemplatePiece::String("xchg rsi, rbx".to_string()) + { + assert_eq!(operands.len(), 4); + let (leaf, eax_place) = match operands[0] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax)); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let ebx_place = match operands[1] { + InlineAsmOperand::Out { reg, late: true, place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::si)); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + let (sub_leaf, ecx_place) = match operands[2] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::cx)); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let edx_place = match operands[3] { + InlineAsmOperand::Out { reg, late: true, place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::dx)); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + + let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf); + + eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); + ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); + ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); + edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); + return; + } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { + // ___chkstk, ___chkstk_ms and __alloca are only used on Windows + crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); + } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" { + crate::trap::trap_unimplemented(fx, "Alloca is not supported"); } let mut slot_size = Size::from_bytes(0); @@ -92,8 +150,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( let inline_asm_index = fx.inline_asm_index; fx.inline_asm_index += 1; - let asm_name = - format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index); + let asm_name = format!("{}__inline_asm_{}", fx.symbol_name, inline_asm_index); let generated_asm = generate_asm_wrapper( &asm_name, @@ -202,7 +259,6 @@ fn call_inline_asm<'tcx>( } let inline_asm_func = fx - .cx .module .declare_function( asm_name, @@ -214,7 +270,7 @@ fn call_inline_asm<'tcx>( }, ) .unwrap(); - let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); + let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(inline_asm_func, asm_name); } diff --git a/src/intrinsics/cpuid.rs b/src/intrinsics/cpuid.rs index b27b0eddfbad6..9de12e759bcc8 100644 --- a/src/intrinsics/cpuid.rs +++ b/src/intrinsics/cpuid.rs @@ -8,7 +8,7 @@ use crate::prelude::*; pub(crate) fn codegen_cpuid_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, leaf: Value, - _subleaf: Value, + _sub_leaf: Value, ) -> (Value, Value, Value, Value) { let leaf_0 = fx.bcx.create_block(); let leaf_1 = fx.bcx.create_block(); diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 83c91f789cd25..ba4ed2162cd5d 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -22,7 +22,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( }; // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` - llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) { + "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) { let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); let lane_ty = fx.clif_type(lane_ty).unwrap(); assert!(lane_count <= 32); @@ -51,7 +51,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32)); ret.write_cvalue(fx, res); }; - llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) { + "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) { let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const"); let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) { 0 => FloatCC::Equal, @@ -81,7 +81,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) }); }; - llvm.x86.sse2.psrli.d, (c a, o imm8) { + "llvm.x86.sse2.psrli.d", (c a, o imm8) { let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { @@ -91,7 +91,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( CValue::by_val(res_lane, res_lane_layout) }); }; - llvm.x86.sse2.pslli.d, (c a, o imm8) { + "llvm.x86.sse2.pslli.d", (c a, o imm8) { let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { @@ -101,7 +101,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( CValue::by_val(res_lane, res_lane_layout) }); }; - llvm.x86.sse2.storeu.dq, (v mem_addr, c a) { + "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) { // FIXME correctly handle the unalignment let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); dest.write_cvalue(fx, a); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 39e047a98f9eb..435737f3a513b 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -8,23 +8,25 @@ mod simd; pub(crate) use cpuid::codegen_cpuid_call; pub(crate) use llvm::codegen_llvm_intrinsic_call; +use rustc_span::symbol::{sym, kw}; +use rustc_middle::ty::print::with_no_trimmed_paths; + use crate::prelude::*; use cranelift_codegen::ir::AtomicRmwOp; -use rustc_middle::ty::print::with_no_trimmed_paths; macro intrinsic_pat { (_) => { _ }, ($name:ident) => { - stringify!($name) + sym::$name + }, + (kw.$name:ident) => { + kw::$name }, ($name:literal) => { - stringify!($name) + $name }, - ($x:ident . $($xs:tt).*) => { - concat!(stringify!($x), ".", intrinsic_pat!($($xs).*)) - } } macro intrinsic_arg { @@ -87,7 +89,7 @@ macro call_intrinsic_match { )*) => { match $intrinsic { $( - stringify!($name) => { + sym::$name => { assert!($substs.is_noop()); if let [$(ref $arg),*] = *$args { let ($($arg,)*) = ( @@ -400,18 +402,17 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let def_id = instance.def_id(); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; + let intrinsic = fx.tcx.item_name(def_id); let ret = match destination { Some((place, _)) => place, None => { // Insert non returning intrinsics here match intrinsic { - "abort" => { + sym::abort => { trap_abort(fx, "Called intrinsic::abort."); } - "transmute" => { + sym::transmute => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); } _ => unimplemented!("unsupported instrinsic {}", intrinsic), @@ -420,7 +421,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( } }; - if intrinsic.starts_with("simd_") { + if intrinsic.as_str().starts_with("simd_") { self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span); let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1); fx.bcx.ins().jump(ret_block, &[]); @@ -470,8 +471,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( sinf64(flt) -> f64 => sin, cosf32(flt) -> f32 => cosf, cosf64(flt) -> f64 => cos, - tanf32(flt) -> f32 => tanf, - tanf64(flt) -> f64 => tan, } intrinsic_match! { @@ -496,12 +495,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( count }; - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::copy_nonoverlapping { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; // NOTE: the volatile variants have src and dst swapped @@ -515,12 +514,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::volatile_copy_nonoverlapping_memory { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; size_of_val, (c ptr) { @@ -552,27 +551,28 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); }; - _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) { + unchecked_add | unchecked_sub | unchecked_div | exact_div | unchecked_rem + | unchecked_shl | unchecked_shr, (c x, c y) { // FIXME trap on overflow let bin_op = match intrinsic { - "unchecked_add" => BinOp::Add, - "unchecked_sub" => BinOp::Sub, - "unchecked_div" | "exact_div" => BinOp::Div, - "unchecked_rem" => BinOp::Rem, - "unchecked_shl" => BinOp::Shl, - "unchecked_shr" => BinOp::Shr, - _ => unreachable!("intrinsic {}", intrinsic), + sym::unchecked_add => BinOp::Add, + sym::unchecked_sub => BinOp::Sub, + sym::unchecked_div | sym::exact_div => BinOp::Div, + sym::unchecked_rem => BinOp::Rem, + sym::unchecked_shl => BinOp::Shl, + sym::unchecked_shr => BinOp::Shr, + _ => unreachable!(), }; let res = crate::num::codegen_int_binop(fx, bin_op, x, y); ret.write_cvalue(fx, res); }; - _ if intrinsic.ends_with("_with_overflow"), (c x, c y) { + add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) { assert_eq!(x.layout().ty, y.layout().ty); let bin_op = match intrinsic { - "add_with_overflow" => BinOp::Add, - "sub_with_overflow" => BinOp::Sub, - "mul_with_overflow" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), + sym::add_with_overflow => BinOp::Add, + sym::sub_with_overflow => BinOp::Sub, + sym::mul_with_overflow => BinOp::Mul, + _ => unreachable!(), }; let res = crate::num::codegen_checked_int_binop( @@ -583,12 +583,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ); ret.write_cvalue(fx, res); }; - _ if intrinsic.starts_with("saturating_"), (c lhs, c rhs) { + saturating_add | saturating_sub, (c lhs, c rhs) { assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { - "saturating_add" => BinOp::Add, - "saturating_sub" => BinOp::Sub, - _ => unreachable!("intrinsic {}", intrinsic), + sym::saturating_add => BinOp::Add, + sym::saturating_sub => BinOp::Sub, + _ => unreachable!(), }; let signed = type_sign(T); @@ -609,15 +609,15 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); let val = match (intrinsic, signed) { - ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val), - ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val), - ("saturating_add", true) => { + (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val), + (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val), + (sym::saturating_add, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); fx.bcx.ins().select(has_overflow, sat_val, val) } - ("saturating_sub", true) => { + (sym::saturating_sub, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); @@ -632,11 +632,21 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; rotate_left, (v x, v y) { let layout = fx.layout_of(T); + let y = if fx.bcx.func.dfg.value_type(y) == types::I128 { + fx.bcx.ins().ireduce(types::I64, y) + } else { + y + }; let res = fx.bcx.ins().rotl(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); }; rotate_right, (v x, v y) { let layout = fx.layout_of(T); + let y = if fx.bcx.func.dfg.value_type(y) == types::I128 { + fx.bcx.ins().ireduce(types::I64, y) + } else { + y + }; let res = fx.bcx.ins().rotr(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); }; @@ -670,7 +680,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let dst_ptr = dst.load_scalar(fx); // FIXME make the memset actually volatile when switching to emit_small_memset // FIXME use emit_small_memset - fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count); + fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count); }; ctlz | ctlz_nonzero, (v arg) { // FIXME trap on `ctlz_nonzero` with zero arg. @@ -806,7 +816,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( return; } - if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { + if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to zero-initialize type `{}`, which is invalid", T), @@ -815,7 +825,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( return; } - if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { + if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to leave type `{}` uninitialized, which is invalid", T), @@ -827,7 +837,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( volatile_load | unaligned_volatile_load, (c ptr) { // Cranelift treats loads as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_load let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); @@ -836,7 +845,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; volatile_store | unaligned_volatile_store, (v ptr, c val) { // Cranelift treats stores as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_store let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); @@ -878,14 +886,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, caller_location); }; - _ if intrinsic.starts_with("atomic_fence"), () { + _ if intrinsic.as_str().starts_with("atomic_fence"), () { fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_singlethreadfence"), () { + _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () { // FIXME use a compiler fence once Cranelift supports it fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_load"), (v ptr) { + _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) { validate_atomic_type!(fx, intrinsic, span, T); let ty = fx.clif_type(T).unwrap(); @@ -894,14 +902,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(T)); ret.write_cvalue(fx, val); }; - _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) { + _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) { validate_atomic_type!(fx, intrinsic, span, val.layout().ty); let val = val.load_scalar(fx); fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); }; - _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) { + _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) { let layout = new.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -913,7 +921,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* + _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* let layout = new.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); @@ -927,7 +935,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, ret_val) }; - _ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) { + _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) { let layout = amount.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -939,7 +947,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) { + _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) { let layout = amount.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -951,7 +959,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_and"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -963,7 +971,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_or"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -975,7 +983,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -989,7 +997,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME https://github.com/bytecodealliance/wasmtime/issues/2647 - _ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1001,7 +1009,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_max"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1013,7 +1021,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1025,7 +1033,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_min"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1037,7 +1045,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1071,7 +1079,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, val); }; - try, (v f, v data, v _catch_fn) { + kw.Try, (v f, v data, v _catch_fn) { // FIXME once unwinding is supported, change this to actually catch panics let f_sig = fx.bcx.func.import_signature(Signature { call_conv: CallConv::triple_default(fx.triple()), @@ -1088,11 +1096,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) { let res = crate::num::codegen_float_binop(fx, match intrinsic { - "fadd_fast" => BinOp::Add, - "fsub_fast" => BinOp::Sub, - "fmul_fast" => BinOp::Mul, - "fdiv_fast" => BinOp::Div, - "frem_fast" => BinOp::Rem, + sym::fadd_fast => BinOp::Add, + sym::fsub_fast => BinOp::Sub, + sym::fmul_fast => BinOp::Mul, + sym::fdiv_fast => BinOp::Div, + sym::frem_fast => BinOp::Rem, _ => unreachable!(), }, x, y); ret.write_cvalue(fx, res); diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index 27fc2abedc7e9..940d2514f7446 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -13,8 +13,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let def_id = instance.def_id(); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; + let intrinsic = fx.tcx.item_name(def_id); intrinsic_match! { fx, intrinsic, substs, args, @@ -65,10 +64,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; // simd_shuffle32(x: T, y: T, idx: [u32; 32]) -> U - _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) { + _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap(); + let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap(); assert_eq!(x.layout(), y.layout()); let layout = x.layout(); diff --git a/src/lib.rs b/src/lib.rs index 720d2a1253445..32f403957025a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)] +#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] #![warn(unreachable_pub)] @@ -23,7 +23,6 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; -use std::str::FromStr; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; @@ -34,9 +33,10 @@ use rustc_middle::ty::query::Providers; use rustc_session::config::OutputFilenames; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; -use crate::constant::ConstantCx; +pub use crate::config::*; use crate::prelude::*; mod abi; @@ -49,6 +49,7 @@ mod cast; mod codegen_i128; mod common; mod compiler_builtins; +mod config; mod constant; mod debuginfo; mod discriminant; @@ -87,7 +88,6 @@ mod prelude { pub(crate) use rustc_index::vec::Idx; - pub(crate) use cranelift_codegen::entity::EntitySet; pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; pub(crate) use cranelift_codegen::ir::function::Function; pub(crate) use cranelift_codegen::ir::types; @@ -119,95 +119,36 @@ impl String> Drop for PrintOnPanic { } } -struct CodegenCx<'m, 'tcx: 'm> { +/// The codegen context holds any information shared between the codegen of individual functions +/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). +struct CodegenCx<'tcx> { tcx: TyCtxt<'tcx>, - module: &'m mut dyn Module, global_asm: String, - constants_cx: ConstantCx, cached_context: Context, - vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, debug_context: Option>, - unwind_context: UnwindContext<'tcx>, + unwind_context: UnwindContext, } -impl<'m, 'tcx> CodegenCx<'m, 'tcx> { +impl<'tcx> CodegenCx<'tcx> { fn new( tcx: TyCtxt<'tcx>, backend_config: BackendConfig, - module: &'m mut dyn Module, + isa: &dyn TargetIsa, debug_info: bool, ) -> Self { - let unwind_context = UnwindContext::new( - tcx, - module.isa(), - matches!(backend_config.codegen_mode, CodegenMode::Aot), - ); - let debug_context = - if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None }; + assert_eq!(pointer_ty(tcx), isa.pointer_type()); + + let unwind_context = + UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot)); + let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None }; CodegenCx { tcx, - module, global_asm: String::new(), - constants_cx: ConstantCx::default(), cached_context: Context::new(), - vtables: FxHashMap::default(), debug_context, unwind_context, } } - - fn finalize(self) -> (String, Option>, UnwindContext<'tcx>) { - self.constants_cx.finalize(self.tcx, self.module); - (self.global_asm, self.debug_context, self.unwind_context) - } -} - -#[derive(Copy, Clone, Debug)] -pub enum CodegenMode { - Aot, - Jit, - JitLazy, -} - -impl Default for CodegenMode { - fn default() -> Self { - CodegenMode::Aot - } -} - -impl FromStr for CodegenMode { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "aot" => Ok(CodegenMode::Aot), - "jit" => Ok(CodegenMode::Jit), - "jit-lazy" => Ok(CodegenMode::JitLazy), - _ => Err(format!("Unknown codegen mode `{}`", s)), - } - } -} - -#[derive(Copy, Clone, Debug, Default)] -pub struct BackendConfig { - pub codegen_mode: CodegenMode, -} - -impl BackendConfig { - fn from_opts(opts: &[String]) -> Result { - let mut config = BackendConfig::default(); - for opt in opts { - if let Some((name, value)) = opt.split_once('=') { - match name { - "mode" => config.codegen_mode = value.parse()?, - _ => return Err(format!("Unknown option `{}`", name)), - } - } else { - return Err(format!("Invalid option `{}`", opt)); - } - } - Ok(config) - } } pub struct CraneliftCodegenBackend { @@ -240,13 +181,23 @@ impl CodegenBackend for CraneliftCodegenBackend { metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box { - let config = if let Some(config) = self.config { + tcx.sess.abort_if_errors(); + let config = if let Some(config) = self.config.clone() { config } else { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) .unwrap_or_else(|err| tcx.sess.fatal(&err)) }; - driver::codegen_crate(tcx, metadata, need_metadata_module, config) + match config.codegen_mode { + CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + #[cfg(feature = "jit")] + let _: ! = driver::jit::run_jit(tcx, config); + + #[cfg(not(feature = "jit"))] + tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } + } } fn join_codegen( @@ -284,7 +235,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple { sess.target.llvm_target.parse().unwrap() } -fn build_isa(sess: &Session) -> Box { +fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); @@ -292,9 +243,8 @@ fn build_isa(sess: &Session) -> Box { let mut flags_builder = settings::builder(); flags_builder.enable("is_pic").unwrap(); flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided - let enable_verifier = - cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok(); - flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap(); + let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" }; + flags_builder.set("enable_verifier", enable_verifier).unwrap(); let tls_model = match target_triple.binary_format { BinaryFormat::Elf => "elf_gd", @@ -322,10 +272,28 @@ fn build_isa(sess: &Session) -> Box { let flags = settings::Flags::new(flags_builder); let variant = cranelift_codegen::isa::BackendVariant::MachInst; - let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); - // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt` - // is interpreted as `bsr`. - isa_builder.enable("nehalem").unwrap(); + + let isa_builder = match sess.opts.cg.target_cpu.as_deref() { + Some("native") => { + let builder = cranelift_native::builder_with_options(variant, true).unwrap(); + builder + } + Some(value) => { + let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + if let Err(_) = builder.enable(value) { + sess.fatal("The specified target cpu isn't currently supported by Cranelift."); + } + builder + } + None => { + let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + // Don't use "haswell" as the default, as it implies `has_lzcnt`. + // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`. + builder.enable("nehalem").unwrap(); + builder + } + }; + isa_builder.finish(flags) } diff --git a/src/linkage.rs b/src/linkage.rs index a564a59f72510..ca853aac15892 100644 --- a/src/linkage.rs +++ b/src/linkage.rs @@ -13,6 +13,7 @@ pub(crate) fn get_clif_linkage( (RLinkage::External, Visibility::Default) => Linkage::Export, (RLinkage::Internal, Visibility::Default) => Linkage::Local, (RLinkage::External, Visibility::Hidden) => Linkage::Hidden, + (RLinkage::WeakAny, Visibility::Default) => Linkage::Preemptible, _ => panic!("{:?} = {:?} {:?}", mono_item, linkage, visibility), } } diff --git a/src/main_shim.rs b/src/main_shim.rs index dc86290b3fe93..d504024a3358e 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -1,6 +1,9 @@ use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_hir::LangItem; +use rustc_middle::ty::subst::GenericArg; +use rustc_middle::ty::AssocKind; use rustc_session::config::EntryFnType; +use rustc_span::symbol::Ident; use crate::prelude::*; @@ -9,9 +12,10 @@ use crate::prelude::*; pub(crate) fn maybe_create_entry_wrapper( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, + is_jit: bool, ) { - let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) { + let (main_def_id, is_main_fn) = match tcx.entry_fn(LOCAL_CRATE) { Some((def_id, entry_ty)) => ( def_id, match entry_ty { @@ -23,18 +27,19 @@ pub(crate) fn maybe_create_entry_wrapper( }; let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - if module.get_name(&*tcx.symbol_name(instance).name).is_none() { + if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() { return; } - create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item); + create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn); fn create_entry_fn( tcx: TyCtxt<'_>, m: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, rust_main_def_id: DefId, - use_start_lang_item: bool, + ignore_lang_start_wrapper: bool, + is_main_fn: bool, ) { let main_ret_ty = tcx.fn_sig(rust_main_def_id).output(); // Given that `main()` has no arguments, @@ -57,9 +62,9 @@ pub(crate) fn maybe_create_entry_wrapper( let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); - let main_name = tcx.symbol_name(instance).name.to_string(); + let main_name = tcx.symbol_name(instance).name; let main_sig = get_function_sig(tcx, m.isa().triple(), instance); - let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap(); + let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap(); let mut ctx = Context::new(); ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig); @@ -74,7 +79,47 @@ pub(crate) fn maybe_create_entry_wrapper( let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func); - let call_inst = if use_start_lang_item { + let result = if is_main_fn && ignore_lang_start_wrapper { + // regular main fn, but ignoring #[lang = "start"] as we are running in the jit + // FIXME set program arguments somehow + let call_inst = bcx.ins().call(main_func_ref, &[]); + let call_results = bcx.func.dfg.inst_results(call_inst).to_owned(); + + let termination_trait = tcx.require_lang_item(LangItem::Termination, None); + let report = tcx + .associated_items(termination_trait) + .find_by_name_and_kind( + tcx, + Ident::from_str("report"), + AssocKind::Fn, + termination_trait, + ) + .unwrap(); + let report = Instance::resolve( + tcx, + ParamEnv::reveal_all(), + report.def_id, + tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()), + ) + .unwrap() + .unwrap(); + + let report_name = tcx.symbol_name(report).name; + let report_sig = get_function_sig(tcx, m.isa().triple(), report); + let report_func_id = + m.declare_function(report_name, Linkage::Import, &report_sig).unwrap(); + let report_func_ref = m.declare_func_in_func(report_func_id, &mut bcx.func); + + // FIXME do proper abi handling instead of expecting the pass mode to be identical + // for returns and arguments. + let report_call_inst = bcx.ins().call(report_func_ref, &call_results); + let res = bcx.func.dfg.inst_results(report_call_inst)[0]; + match m.target_config().pointer_type() { + types::I32 => res, + types::I64 => bcx.ins().sextend(types::I64, res), + _ => unimplemented!("16bit systems are not yet supported"), + } + } else if is_main_fn { let start_def_id = tcx.require_lang_item(LangItem::Start, None); let start_instance = Instance::resolve( tcx, @@ -90,13 +135,14 @@ pub(crate) fn maybe_create_entry_wrapper( let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref); let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func); - bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]) + let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] } else { // using user-defined start fn - bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]) + let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] }; - let result = bcx.inst_results(call_inst)[0]; bcx.ins().return_(&[result]); bcx.seal_all_blocks(); bcx.finalize(); diff --git a/src/metadata.rs b/src/metadata.rs index dbdc8cbad44c4..882232fde09d2 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -8,13 +8,24 @@ use rustc_data_structures::memmap::Mmap; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; -use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; +use rustc_middle::middle::cstore::MetadataLoader; use rustc_middle::ty::TyCtxt; -use rustc_session::config; use rustc_target::spec::Target; use crate::backend::WriteMetadata; +/// The metadata loader used by cg_clif. +/// +/// The metadata is stored in the same format as cg_llvm. +/// +/// # Metadata location +/// +///
+///
rlib
+///
The metadata can be found in the `lib.rmeta` file inside of the ar archive.
+///
dylib
+///
The metadata can be found in the `.rustc` section of the shared library.
+///
pub(crate) struct CraneliftMetadataLoader; fn load_metadata_with( @@ -58,54 +69,16 @@ impl MetadataLoader for CraneliftMetadataLoader { } // Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112 -pub(crate) fn write_metadata( - tcx: TyCtxt<'_>, - product: &mut P, -) -> EncodedMetadata { +pub(crate) fn write_metadata(tcx: TyCtxt<'_>, object: &mut O) { use snap::write::FrameEncoder; use std::io::Write; - #[derive(PartialEq, Eq, PartialOrd, Ord)] - enum MetadataKind { - None, - Uncompressed, - Compressed, - } - - let kind = tcx - .sess - .crate_types() - .iter() - .map(|ty| match *ty { - config::CrateType::Executable - | config::CrateType::Staticlib - | config::CrateType::Cdylib => MetadataKind::None, - - config::CrateType::Rlib => MetadataKind::Uncompressed, - - config::CrateType::Dylib | config::CrateType::ProcMacro => MetadataKind::Compressed, - }) - .max() - .unwrap_or(MetadataKind::None); - - if kind == MetadataKind::None { - return EncodedMetadata::new(); - } - let metadata = tcx.encode_metadata(); - if kind == MetadataKind::Uncompressed { - return metadata; - } - - assert!(kind == MetadataKind::Compressed); let mut compressed = tcx.metadata_encoding_version(); FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap(); - product.add_rustc_section( + object.add_rustc_section( rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx), compressed, - tcx.sess.target.is_like_osx, ); - - metadata } diff --git a/src/num.rs b/src/num.rs index 2ebf30da2d8ba..b6d378a5fe10a 100644 --- a/src/num.rs +++ b/src/num.rs @@ -271,14 +271,17 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let val_hi = fx.bcx.ins().umulhi(lhs, rhs); fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0) } else { + // Based on LLVM's instruction sequence for compiling + // a.checked_mul(b).is_some() to riscv64gc: + // mulh a2, a0, a1 + // mul a0, a0, a1 + // srai a0, a0, 63 + // xor a0, a0, a2 + // snez a0, a0 let val_hi = fx.bcx.ins().smulhi(lhs, rhs); - let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0); - let not_all_ones = fx.bcx.ins().icmp_imm( - IntCC::NotEqual, - val_hi, - u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64, - ); - fx.bcx.ins().band(not_all_zero, not_all_ones) + let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1)); + let xor = fx.bcx.ins().bxor(val_hi, val_sign); + fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0) }; (val, has_overflow) } diff --git a/src/optimize/code_layout.rs b/src/optimize/code_layout.rs deleted file mode 100644 index ca9ff15ec10ff..0000000000000 --- a/src/optimize/code_layout.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! This optimization moves cold code to the end of the function. -//! -//! Some code is executed much less often than other code. For example panicking or the -//! landingpads for unwinding. By moving this cold code to the end of the function the average -//! amount of jumps is reduced and the code locality is improved. -//! -//! # Undefined behaviour -//! -//! This optimization doesn't assume anything that isn't already assumed by Cranelift itself. - -use crate::prelude::*; - -pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet) { - // FIXME Move the block in place instead of remove and append once - // bytecodealliance/cranelift#1339 is implemented. - - let mut block_insts = FxHashMap::default(); - for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) { - let insts = ctx.func.layout.block_insts(block).collect::>(); - for &inst in &insts { - ctx.func.layout.remove_inst(inst); - } - block_insts.insert(block, insts); - ctx.func.layout.remove_block(block); - } - - // And then append them at the back again. - for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) { - ctx.func.layout.append_block(block); - for inst in block_insts.remove(&block).unwrap() { - ctx.func.layout.append_inst(inst, block); - } - } -} diff --git a/src/optimize/mod.rs b/src/optimize/mod.rs index 389f50e797e89..137fb5f77313c 100644 --- a/src/optimize/mod.rs +++ b/src/optimize/mod.rs @@ -2,29 +2,16 @@ use crate::prelude::*; -mod code_layout; pub(crate) mod peephole; -mod stack2reg; pub(crate) fn optimize_function<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, ctx: &mut Context, - cold_blocks: &EntitySet, clif_comments: &mut crate::pretty_clif::CommentWriter, ) { - // The code_layout optimization is very cheap. - self::code_layout::optimize_function(ctx, cold_blocks); + // FIXME classify optimizations over opt levels once we have more - if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No { - return; // FIXME classify optimizations over opt levels - } - - // FIXME(#1142) stack2reg miscompiles lewton - if false { - self::stack2reg::optimize_function(ctx, clif_comments); - } - - crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments); + crate::pretty_clif::write_clif_file(tcx, "preopt", None, instance, &ctx, &*clif_comments); crate::base::verify_func(tcx, &*clif_comments, &ctx.func); } diff --git a/src/optimize/stack2reg.rs b/src/optimize/stack2reg.rs deleted file mode 100644 index 8bb02a3e55854..0000000000000 --- a/src/optimize/stack2reg.rs +++ /dev/null @@ -1,486 +0,0 @@ -//! This optimization replaces stack accesses with SSA variables and removes dead stores when possible. -//! -//! # Undefined behaviour -//! -//! This optimization is based on the assumption that stack slots which don't have their address -//! leaked through `stack_addr` are only accessed using `stack_load` and `stack_store` in the -//! function which has the stack slots. This optimization also assumes that stack slot accesses -//! are never out of bounds. If these assumptions are not correct, then this optimization may remove -//! `stack_store` instruction incorrectly, or incorrectly use a previously stored value as the value -//! being loaded by a `stack_load`. - -use std::collections::BTreeMap; -use std::fmt; -use std::ops::Not; - -use rustc_data_structures::fx::FxHashSet; - -use cranelift_codegen::cursor::{Cursor, FuncCursor}; -use cranelift_codegen::ir::immediates::Offset32; -use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef}; - -use crate::prelude::*; - -/// Workaround for `StackSlot` not implementing `Ord`. -#[derive(Copy, Clone, PartialEq, Eq)] -struct OrdStackSlot(StackSlot); - -impl fmt::Debug for OrdStackSlot { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.0) - } -} - -impl PartialOrd for OrdStackSlot { - fn partial_cmp(&self, rhs: &Self) -> Option { - self.0.as_u32().partial_cmp(&rhs.0.as_u32()) - } -} - -impl Ord for OrdStackSlot { - fn cmp(&self, rhs: &Self) -> std::cmp::Ordering { - self.0.as_u32().cmp(&rhs.0.as_u32()) - } -} - -#[derive(Debug, Default)] -struct StackSlotUsage { - stack_addr: FxHashSet, - stack_load: FxHashSet, - stack_store: FxHashSet, -} - -impl StackSlotUsage { - fn potential_stores_for_load(&self, ctx: &Context, load: Inst) -> Vec { - self.stack_store - .iter() - .cloned() - .filter(|&store| { - match spatial_overlap(&ctx.func, store, load) { - SpatialOverlap::No => false, // Can never be the source of the loaded value. - SpatialOverlap::Partial | SpatialOverlap::Full => true, - } - }) - .filter(|&store| { - match temporal_order(ctx, store, load) { - TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. - TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, - } - }) - .collect::>() - } - - fn potential_loads_of_store(&self, ctx: &Context, store: Inst) -> Vec { - self.stack_load - .iter() - .cloned() - .filter(|&load| { - match spatial_overlap(&ctx.func, store, load) { - SpatialOverlap::No => false, // Can never be the source of the loaded value. - SpatialOverlap::Partial | SpatialOverlap::Full => true, - } - }) - .filter(|&load| { - match temporal_order(ctx, store, load) { - TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. - TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, - } - }) - .collect::>() - } - - fn remove_unused_stack_addr(func: &mut Function, inst: Inst) { - func.dfg.detach_results(inst); - func.dfg.replace(inst).nop(); - } - - fn remove_unused_load(func: &mut Function, load: Inst) { - func.dfg.detach_results(load); - func.dfg.replace(load).nop(); - } - - fn remove_dead_store(&mut self, func: &mut Function, store: Inst) { - func.dfg.replace(store).nop(); - self.stack_store.remove(&store); - } - - fn change_load_to_alias(&mut self, func: &mut Function, load: Inst, value: Value) { - let loaded_value = func.dfg.inst_results(load)[0]; - let loaded_type = func.dfg.value_type(loaded_value); - - if func.dfg.value_type(value) == loaded_type { - func.dfg.detach_results(load); - func.dfg.replace(load).nop(); - func.dfg.change_to_alias(loaded_value, value); - } else { - func.dfg.replace(load).bitcast(loaded_type, value); - } - - self.stack_load.remove(&load); - } -} - -struct OptimizeContext<'a> { - ctx: &'a mut Context, - stack_slot_usage_map: BTreeMap, -} - -impl<'a> OptimizeContext<'a> { - fn for_context(ctx: &'a mut Context) -> Self { - ctx.flowgraph(); // Compute cfg and domtree. - - // Record all stack_addr, stack_load and stack_store instructions. - let mut stack_slot_usage_map = BTreeMap::::new(); - - let mut cursor = FuncCursor::new(&mut ctx.func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - match cursor.func.dfg[inst] { - InstructionData::StackLoad { - opcode: Opcode::StackAddr, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_addr - .insert(inst); - } - InstructionData::StackLoad { - opcode: Opcode::StackLoad, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_load - .insert(inst); - } - InstructionData::StackStore { - opcode: Opcode::StackStore, - arg: _, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_store - .insert(inst); - } - _ => {} - } - } - } - - OptimizeContext { ctx, stack_slot_usage_map } - } -} - -pub(super) fn optimize_function( - ctx: &mut Context, - clif_comments: &mut crate::pretty_clif::CommentWriter, -) { - combine_stack_addr_with_load_store(&mut ctx.func); - - let mut opt_ctx = OptimizeContext::for_context(ctx); - - // FIXME Repeat following instructions until fixpoint. - - remove_unused_stack_addr_and_stack_load(&mut opt_ctx); - - if clif_comments.enabled() { - for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map { - clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage)); - } - } - - for (stack_slot, users) in opt_ctx.stack_slot_usage_map.iter_mut() { - if users.stack_addr.is_empty().not() { - // Stack addr leaked; there may be unknown loads and stores. - // FIXME use stacked borrows to optimize - continue; - } - - for load in users.stack_load.clone().into_iter() { - let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load); - - if clif_comments.enabled() { - for &store in &potential_stores { - clif_comments.add_comment( - load, - format!( - "Potential store -> load forwarding {} -> {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(store, None), - opt_ctx.ctx.func.dfg.display_inst(load, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); - } - } - - match *potential_stores { - [] => { - if clif_comments.enabled() { - clif_comments - .add_comment(load, "[BUG?] Reading uninitialized memory".to_string()); - } - } - [store] - if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full - && temporal_order(&opt_ctx.ctx, store, load) - == TemporalOrder::DefinitivelyBefore => - { - // Only one store could have been the origin of the value. - let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0]; - - if clif_comments.enabled() { - clif_comments.add_comment( - load, - format!("Store to load forward {} -> {}", store, load), - ); - } - - users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value); - } - _ => {} // FIXME implement this - } - } - - for store in users.stack_store.clone().into_iter() { - let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store); - - if clif_comments.enabled() { - for &load in &potential_loads { - clif_comments.add_comment( - store, - format!( - "Potential load from store {} <- {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(load, None), - opt_ctx.ctx.func.dfg.display_inst(store, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); - } - } - - if potential_loads.is_empty() { - // Never loaded; can safely remove all stores and the stack slot. - // FIXME also remove stores when there is always a next store before a load. - - if clif_comments.enabled() { - clif_comments.add_comment( - store, - format!( - "Remove dead stack store {} of {}", - opt_ctx.ctx.func.dfg.display_inst(store, None), - stack_slot.0 - ), - ); - } - - users.remove_dead_store(&mut opt_ctx.ctx.func, store); - } - } - - if users.stack_store.is_empty() && users.stack_load.is_empty() { - opt_ctx.ctx.func.stack_slots[stack_slot.0].size = 0; - } - } -} - -fn combine_stack_addr_with_load_store(func: &mut Function) { - // Turn load and store into stack_load and stack_store when possible. - let mut cursor = FuncCursor::new(func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - match cursor.func.dfg[inst] { - InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => { - if cursor.func.dfg.ctrl_typevar(inst) == types::I128 - || cursor.func.dfg.ctrl_typevar(inst).is_vector() - { - continue; // WORKAROUD: stack_load.i128 not yet implemented - } - if let Some((stack_slot, stack_addr_offset)) = - try_get_stack_slot_and_offset_for_addr(cursor.func, addr) - { - if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) - { - let ty = cursor.func.dfg.ctrl_typevar(inst); - cursor.func.dfg.replace(inst).stack_load( - ty, - stack_slot, - combined_offset, - ); - } - } - } - InstructionData::Store { - opcode: Opcode::Store, - args: [value, addr], - flags: _, - offset, - } => { - if cursor.func.dfg.ctrl_typevar(inst) == types::I128 - || cursor.func.dfg.ctrl_typevar(inst).is_vector() - { - continue; // WORKAROUND: stack_store.i128 not yet implemented - } - if let Some((stack_slot, stack_addr_offset)) = - try_get_stack_slot_and_offset_for_addr(cursor.func, addr) - { - if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) - { - cursor.func.dfg.replace(inst).stack_store( - value, - stack_slot, - combined_offset, - ); - } - } - } - _ => {} - } - } - } -} - -fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) { - // FIXME incrementally rebuild on each call? - let mut stack_addr_load_insts_users = FxHashMap::>::default(); - - let mut cursor = FuncCursor::new(&mut opt_ctx.ctx.func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - for &arg in cursor.func.dfg.inst_args(inst) { - if let ValueDef::Result(arg_origin, 0) = cursor.func.dfg.value_def(arg) { - match cursor.func.dfg[arg_origin].opcode() { - Opcode::StackAddr | Opcode::StackLoad => { - stack_addr_load_insts_users - .entry(arg_origin) - .or_insert_with(FxHashSet::default) - .insert(inst); - } - _ => {} - } - } - } - } - } - - #[cfg(debug_assertions)] - for inst in stack_addr_load_insts_users.keys() { - let mut is_recorded_stack_addr_or_stack_load = false; - for stack_slot_users in opt_ctx.stack_slot_usage_map.values() { - is_recorded_stack_addr_or_stack_load |= stack_slot_users.stack_addr.contains(inst) - || stack_slot_users.stack_load.contains(inst); - } - assert!(is_recorded_stack_addr_or_stack_load); - } - - // Replace all unused stack_addr and stack_load instructions with nop. - let mut func = &mut opt_ctx.ctx.func; - - for stack_slot_users in opt_ctx.stack_slot_usage_map.values_mut() { - stack_slot_users - .stack_addr - .drain_filter(|inst| { - stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true) - }) - .for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst)); - - stack_slot_users - .stack_load - .drain_filter(|inst| { - stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true) - }) - .for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst)); - } -} - -fn try_get_stack_slot_and_offset_for_addr( - func: &Function, - addr: Value, -) -> Option<(StackSlot, Offset32)> { - if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) { - if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } = - func.dfg[addr_inst] - { - return Some((stack_slot, offset)); - } - } - None -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum SpatialOverlap { - No, - Partial, - Full, -} - -fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap { - fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) { - match func.dfg[inst] { - InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } - | InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset } - | InstructionData::StackStore { - opcode: Opcode::StackStore, - stack_slot, - offset, - arg: _, - } => (stack_slot, offset, func.dfg.ctrl_typevar(inst).bytes()), - _ => unreachable!("{:?}", func.dfg[inst]), - } - } - - debug_assert_ne!(src, dest); - - let (src_ss, src_offset, src_size) = inst_info(func, src); - let (dest_ss, dest_offset, dest_size) = inst_info(func, dest); - - if src_ss != dest_ss { - return SpatialOverlap::No; - } - - if src_offset == dest_offset && src_size == dest_size { - return SpatialOverlap::Full; - } - - let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into(); - let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into(); - if src_end <= dest_offset.into() || dest_end <= src_offset.into() { - return SpatialOverlap::No; - } - - SpatialOverlap::Partial -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum TemporalOrder { - /// `src` will never be executed before `dest`. - NeverBefore, - - /// `src` may be executed before `dest`. - MaybeBefore, - - /// `src` will always be executed before `dest`. - /// There may still be other instructions in between. - DefinitivelyBefore, -} - -fn temporal_order(ctx: &Context, src: Inst, dest: Inst) -> TemporalOrder { - debug_assert_ne!(src, dest); - - if ctx.domtree.dominates(src, dest, &ctx.func.layout) { - TemporalOrder::DefinitivelyBefore - } else if ctx.domtree.dominates(src, dest, &ctx.func.layout) { - TemporalOrder::NeverBefore - } else { - TemporalOrder::MaybeBefore - } -} diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index d22ea3772eee7..158811c5eaf49 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -207,7 +207,7 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { pub(crate) fn write_ir_file( tcx: TyCtxt<'_>, - name: &str, + name: impl FnOnce() -> String, write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, ) { if !should_write_ir(tcx) { @@ -222,7 +222,7 @@ pub(crate) fn write_ir_file( res @ Err(_) => res.unwrap(), } - let clif_file_name = clif_output_dir.join(name); + let clif_file_name = clif_output_dir.join(name()); let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file)); if let Err(err) = res { @@ -238,30 +238,31 @@ pub(crate) fn write_clif_file<'tcx>( context: &cranelift_codegen::Context, mut clif_comments: &CommentWriter, ) { - write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| { - let value_ranges = - isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges")); + write_ir_file( + tcx, + || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), + |file| { + let value_ranges = isa + .map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges")); - let mut clif = String::new(); - cranelift_codegen::write::decorate_function( - &mut clif_comments, - &mut clif, - &context.func, - &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa(tcx.sess)), - value_ranges: value_ranges.as_ref(), - }, - ) - .unwrap(); + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() }, + ) + .unwrap(); - writeln!(file, "test compile")?; - writeln!(file, "set is_pic")?; - writeln!(file, "set enable_simd")?; - writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; - writeln!(file)?; - file.write_all(clif.as_bytes())?; - Ok(()) - }); + writeln!(file, "test compile")?; + writeln!(file, "set is_pic")?; + writeln!(file, "set enable_simd")?; + writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }, + ); } impl fmt::Debug for FunctionCx<'_, '_, '_> { diff --git a/src/trap.rs b/src/trap.rs index 1ab0703e981e7..819c8b51558a0 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -4,7 +4,6 @@ use crate::prelude::*; fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { let puts = fx - .cx .module .declare_function( "puts", @@ -16,13 +15,12 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { }, ) .unwrap(); - let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func); + let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(puts, "puts"); } - let symbol_name = fx.tcx.symbol_name(fx.instance); - let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, symbol_name, msg); + let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg); let msg_ptr = fx.anonymous_str("trap", &real_msg); fx.bcx.ins().call(puts, &[msg_ptr]); } diff --git a/src/value_and_place.rs b/src/value_and_place.rs index b97d39009847a..9a572c3501f92 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -554,7 +554,7 @@ impl<'tcx> CPlace<'tcx> { let src_align = src_layout.align.abi.bytes() as u8; let dst_align = dst_layout.align.abi.bytes() as u8; fx.bcx.emit_small_memory_copy( - fx.cx.module.target_config(), + fx.module.target_config(), to_addr, from_addr, size, diff --git a/src/vtable.rs b/src/vtable.rs index 9053d1aa1b05a..bbf07ffc85dbd 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -72,15 +72,15 @@ pub(crate) fn get_vtable<'tcx>( layout: TyAndLayout<'tcx>, trait_ref: Option>, ) -> Value { - let data_id = if let Some(data_id) = fx.cx.vtables.get(&(layout.ty, trait_ref)) { + let data_id = if let Some(data_id) = fx.vtables.get(&(layout.ty, trait_ref)) { *data_id } else { let data_id = build_vtable(fx, layout, trait_ref); - fx.cx.vtables.insert((layout.ty, trait_ref), data_id); + fx.vtables.insert((layout.ty, trait_ref), data_id); data_id }; - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } @@ -94,7 +94,7 @@ fn build_vtable<'tcx>( let drop_in_place_fn = import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx), ); @@ -111,7 +111,7 @@ fn build_vtable<'tcx>( opt_mth.map(|(def_id, substs)| { import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs) .unwrap() .polymorphize(fx.tcx), @@ -132,34 +132,16 @@ fn build_vtable<'tcx>( for (i, component) in components.into_iter().enumerate() { if let Some(func_id) = component { - let func_ref = fx.cx.module.declare_func_in_data(func_id, &mut data_ctx); + let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx); data_ctx.write_function_addr((i * usize_size) as u32, func_ref); } } data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes()); - let data_id = fx - .cx - .module - .declare_data( - &format!( - "__vtable.{}.for.{:?}.{}", - trait_ref - .as_ref() - .map(|trait_ref| format!("{:?}", trait_ref.skip_binder()).into()) - .unwrap_or(std::borrow::Cow::Borrowed("???")), - layout.ty, - fx.cx.vtables.len(), - ), - Linkage::Local, - false, - false, - ) - .unwrap(); - - // FIXME don't duplicate definitions in lazy jit mode - let _ = fx.cx.module.define_data(data_id, &data_ctx); + let data_id = fx.module.declare_anonymous_data(false, false).unwrap(); + + fx.module.define_data(data_id, &data_ctx).unwrap(); data_id } From df68e5ac73f23e1006b811bdc50b8c2ede0e0bbf Mon Sep 17 00:00:00 2001 From: Erin Power Date: Fri, 30 Apr 2021 15:27:05 +0200 Subject: [PATCH 04/43] [cg_clif] Fix run_jit from sync --- src/driver/jit.rs | 56 +++++------------------------------------------ 1 file changed, 5 insertions(+), 51 deletions(-) diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 956207c5b7cfe..53c93f6a9ddca 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -8,6 +8,7 @@ use std::os::raw::{c_char, c_int}; use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; +use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; @@ -138,57 +139,10 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { *lazy_jit_state = Some(JitState { backend_config, jit_module }); }); - let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); - let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - - match entry_ty { - EntryFnType::Main => { - // FIXME set program arguments somehow - - let main_sig = Signature { - params: vec![], - returns: vec![], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let main_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig) - .unwrap(); - let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) }; - f(); - std::process::exit(0); - } - EntryFnType::Start => { - let start_sig = Signature { - params: vec![ - AbiParam::new(jit_module.target_config().pointer_type()), - AbiParam::new(jit_module.target_config().pointer_type()), - ], - returns: vec![AbiParam::new( - jit_module.target_config().pointer_type(), /*isize*/ - )], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let start_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig) - .unwrap(); - let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn(c_int, *const *const c_char) -> c_int = - unsafe { ::std::mem::transmute(finalized_start) }; - let ret = f(args.len() as c_int, argv.as_ptr()); - std::process::exit(ret); - } - } + let f: extern "C" fn(c_int, *const *const c_char) -> c_int = + unsafe { ::std::mem::transmute(finalized_start) }; + let ret = f(args.len() as c_int, argv.as_ptr()); + std::process::exit(ret); } #[no_mangle] From 0a1b87c9ce295030a373cc4e37cac7e89294903f Mon Sep 17 00:00:00 2001 From: Erin Power Date: Fri, 30 Apr 2021 18:46:59 +0200 Subject: [PATCH 05/43] Merge commit '15c8d31392b9fbab3b3368b67acc4bbe5983115a' into cranelift-rebase --- src/debuginfo/unwind.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index ca7083cccb88f..d1251e749f31f 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -71,7 +71,10 @@ impl UnwindContext { } } - #[cfg(feature = "jit")] + #[cfg(all(feature = "jit", windows))] + pub(crate) unsafe fn register_jit(self, _jit_module: &cranelift_jit::JITModule) {} + + #[cfg(all(feature = "jit", not(windows)))] pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) { let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); From 55dbf17bb57c1d05e0cb04c21e43ad3dc7b01858 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 28 Mar 2021 22:14:09 +0200 Subject: [PATCH 06/43] Pass target_cpu to LinkerInfo::new instead of link_binary This is one step towards separating the linking code from codegen backends --- src/driver/aot.rs | 2 +- src/lib.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index ed3bdedddced5..c7121b9386125 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -295,7 +295,7 @@ pub(super) fn run_aot( metadata_module, metadata, windows_subsystem, - linker_info: LinkerInfo::new(tcx), + linker_info: LinkerInfo::new(tcx, crate::target_triple(tcx.sess).to_string()), crate_info: CrateInfo::new(tcx), }, work_products, diff --git a/src/lib.rs b/src/lib.rs index 720d2a1253445..75b7b9bbeb567 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -267,13 +267,11 @@ impl CodegenBackend for CraneliftCodegenBackend { ) -> Result<(), ErrorReported> { use rustc_codegen_ssa::back::link::link_binary; - let target_cpu = crate::target_triple(sess).to_string(); link_binary::>( sess, &codegen_results, outputs, &codegen_results.crate_name.as_str(), - &target_cpu, ); Ok(()) From e7f930123989b4d47b2bb5efcedd1592cdcd6ad0 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 4 May 2021 13:55:04 +0200 Subject: [PATCH 07/43] Rustup to rustc 1.54.0-nightly (716394d65 2021-05-03) --- build_sysroot/Cargo.lock | 6 +++--- rust-toolchain | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index e058a972ead3c..dd5228d3ba4ca 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -56,7 +56,7 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.40" +version = "0.1.42" dependencies = [ "rustc-std-workspace-core", ] @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" +checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce" dependencies = [ "compiler_builtins", "rustc-std-workspace-core", diff --git a/rust-toolchain b/rust-toolchain index 5442e3345aa91..70543cb6c6637 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-04-28" +channel = "nightly-2021-05-04" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 7a61ec3007267e6cb4134b8042fa30bee8f2e88a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 4 May 2021 14:01:22 +0200 Subject: [PATCH 08/43] Add missing change --- build_sysroot/prepare_sysroot_src.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh index f7fcef1077410..2b0feb93d4dea 100755 --- a/build_sysroot/prepare_sysroot_src.sh +++ b/build_sysroot/prepare_sysroot_src.sh @@ -32,7 +32,7 @@ popd git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned" pushd compiler-builtins git checkout -- . -git checkout 0.1.40 +git checkout 0.1.42 git apply ../../crate_patches/000*-compiler-builtins-*.patch popd From 6266e9d907ec2bfd90e0bf15dd0f3411430c612a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 4 May 2021 14:13:07 +0200 Subject: [PATCH 09/43] Update rust bootstrap patches --- scripts/setup_rust_fork.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index 4821a07ac5d5d..e342963de15d1 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -24,18 +24,6 @@ index 5bd1147cad5..10d68a2ff14 100644 + [patch."https://github.com/rust-lang/rust-clippy"] clippy_lints = { path = "src/tools/clippy/clippy_lints" } -diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml -index 23e689fcae7..5f077b765b6 100644 ---- a/compiler/rustc_data_structures/Cargo.toml -+++ b/compiler/rustc_data_structures/Cargo.toml -@@ -32,7 +32,6 @@ tempfile = "3.0.5" - - [dependencies.parking_lot] - version = "0.11" --features = ["nightly"] - - [target.'cfg(windows)'.dependencies] - winapi = { version = "0.3", features = ["fileapi", "psapi"] } diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml index d95b5b7f17f..00b6f0e3635 100644 --- a/library/alloc/Cargo.toml @@ -45,7 +33,7 @@ index d95b5b7f17f..00b6f0e3635 100644 [dependencies] core = { path = "../core" } -compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] } -+compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std', 'no-asm'] } ++compiler_builtins = { version = "0.1.42", features = ['rustc-dep-of-std', 'no-asm'] } [dev-dependencies] rand = "0.7" From 20600f1df9114ee2850ec92bbae15a1408136e0b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 4 May 2021 14:16:59 +0200 Subject: [PATCH 10/43] Force deprecation warning that was promoted to an error back to a warning --- scripts/tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tests.sh b/scripts/tests.sh index 3afcea8f06bd6..0d99d2c507c95 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -116,6 +116,7 @@ function extended_sysroot_tests() { pushd regex echo "[TEST] rust-lang/regex example shootout-regex-dna" cargo clean + export RUSTFLAGS="$RUSTFLAGS --cap-lints warn" # newer aho_corasick versions throw a deprecation warning # Make sure `[codegen mono items] start` doesn't poison the diff ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then From 8b693579877ed67e59dfad5a82d3f2e05d3383d0 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 4 May 2021 16:41:48 +0200 Subject: [PATCH 11/43] Rustfmt --- src/intrinsics/mod.rs | 2 +- src/lib.rs | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 435737f3a513b..52896fc7127e8 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -8,8 +8,8 @@ mod simd; pub(crate) use cpuid::codegen_cpuid_call; pub(crate) use llvm::codegen_llvm_intrinsic_call; -use rustc_span::symbol::{sym, kw}; use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_span::symbol::{kw, sym}; use crate::prelude::*; use cranelift_codegen::ir::AtomicRmwOp; diff --git a/src/lib.rs b/src/lib.rs index 32f403957025a..61b6316b973df 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -279,21 +279,23 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box { - let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + let mut builder = + cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); if let Err(_) = builder.enable(value) { sess.fatal("The specified target cpu isn't currently supported by Cranelift."); } builder } None => { - let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + let mut builder = + cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); // Don't use "haswell" as the default, as it implies `has_lzcnt`. // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`. builder.enable("nehalem").unwrap(); builder } }; - + isa_builder.finish(flags) } From fea01cfaaa4b232c9eaf45d9b07030580c0abaaf Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 4 May 2021 18:54:22 +0200 Subject: [PATCH 12/43] Update Cranelift This removes all the byteorder and thiserror dependencies. This results in no proc macros being necessary anymore. --- Cargo.lock | 84 +++++------------------------------------- Cargo.toml | 8 ---- src/value_and_place.rs | 1 + 3 files changed, 11 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6792def56796..09bb37292ef3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,12 +25,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" -[[package]] -name = "byteorder" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" - [[package]] name = "cfg-if" version = "1.0.0" @@ -40,7 +34,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-entity", ] @@ -48,9 +42,8 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ - "byteorder", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", @@ -60,13 +53,12 @@ dependencies = [ "regalloc", "smallvec", "target-lexicon", - "thiserror", ] [[package]] name = "cranelift-codegen-meta" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -75,17 +67,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" [[package]] name = "cranelift-entity" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" [[package]] name = "cranelift-frontend" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-codegen", "log", @@ -96,7 +88,7 @@ dependencies = [ [[package]] name = "cranelift-jit" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "anyhow", "cranelift-codegen", @@ -114,19 +106,18 @@ dependencies = [ [[package]] name = "cranelift-module" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "anyhow", "cranelift-codegen", "cranelift-entity", "log", - "thiserror", ] [[package]] name = "cranelift-native" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -135,7 +126,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "anyhow", "cranelift-codegen", @@ -250,24 +241,6 @@ dependencies = [ "indexmap", ] -[[package]] -name = "proc-macro2" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" -dependencies = [ - "proc-macro2", -] - [[package]] name = "regalloc" version = "0.0.31" @@ -322,49 +295,12 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" -[[package]] -name = "syn" -version = "1.0.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - [[package]] name = "target-lexicon" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64ae3b39281e4b14b8123bdbaddd472b7dfe215e444181f2f9d2443c2444f834" -[[package]] -name = "thiserror" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "unicode-xid" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 2789207c65581..ff1433d1cb632 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,13 +70,5 @@ debug = false opt-level = 0 debug = false -[profile.dev.package.syn] -opt-level = 0 -debug = false - -[profile.release.package.syn] -opt-level = 0 -debug = false - [package.metadata.rust-analyzer] rustc_private = true diff --git a/src/value_and_place.rs b/src/value_and_place.rs index 9a572c3501f92..171f39805f896 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -561,6 +561,7 @@ impl<'tcx> CPlace<'tcx> { dst_align, src_align, true, + MemFlags::trusted(), ); } CValueInner::ByRef(_, Some(_)) => todo!(), From ac5272b3c7cd929b2144cfc244271e6e3d1d64ee Mon Sep 17 00:00:00 2001 From: Andy Wang Date: Sat, 10 Apr 2021 13:48:06 +0100 Subject: [PATCH 13/43] Use RealFileName for Session::working_dir as it may also be remapped --- src/debuginfo/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debuginfo/mod.rs b/src/debuginfo/mod.rs index dc8bc8d9cb741..10d3651789b3a 100644 --- a/src/debuginfo/mod.rs +++ b/src/debuginfo/mod.rs @@ -64,7 +64,7 @@ impl<'tcx> DebugContext<'tcx> { // FIXME: how to get version when building out of tree? // Normally this would use option_env!("CFG_VERSION"). let producer = format!("cg_clif (rustc {})", "unknown version"); - let comp_dir = tcx.sess.working_dir.0.to_string_lossy().into_owned(); + let comp_dir = tcx.sess.working_dir.stable_name().to_string_lossy().into_owned(); let (name, file_info) = match tcx.sess.local_crate_source_file.clone() { Some(path) => { let name = path.to_string_lossy().into_owned(); From 5de83e7bd1929d6e18c19dc4430bdcececc905c2 Mon Sep 17 00:00:00 2001 From: Andy Wang Date: Wed, 14 Apr 2021 14:12:39 +0100 Subject: [PATCH 14/43] Revamp RealFileName public methods --- src/debuginfo/line_info.rs | 2 +- src/debuginfo/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/debuginfo/line_info.rs b/src/debuginfo/line_info.rs index 8578ab33ced68..b3f700bc46794 100644 --- a/src/debuginfo/line_info.rs +++ b/src/debuginfo/line_info.rs @@ -66,7 +66,7 @@ fn line_program_add_file( ) -> FileId { match &file.name { FileName::Real(path) => { - let (dir_path, file_name) = split_path_dir_and_file(path.stable_name()); + let (dir_path, file_name) = split_path_dir_and_file(path.remapped_path_if_available()); let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str()); let file_name = osstr_as_utf8_bytes(file_name); diff --git a/src/debuginfo/mod.rs b/src/debuginfo/mod.rs index 10d3651789b3a..61e54a76f29ba 100644 --- a/src/debuginfo/mod.rs +++ b/src/debuginfo/mod.rs @@ -64,7 +64,7 @@ impl<'tcx> DebugContext<'tcx> { // FIXME: how to get version when building out of tree? // Normally this would use option_env!("CFG_VERSION"). let producer = format!("cg_clif (rustc {})", "unknown version"); - let comp_dir = tcx.sess.working_dir.stable_name().to_string_lossy().into_owned(); + let comp_dir = tcx.sess.working_dir.to_string_lossy(false).into_owned(); let (name, file_info) = match tcx.sess.local_crate_source_file.clone() { Some(path) => { let name = path.to_string_lossy().into_owned(); From e162eeb1f0492822cc810decee23d6d9b9c084f0 Mon Sep 17 00:00:00 2001 From: Andy Wang Date: Mon, 19 Apr 2021 23:27:02 +0100 Subject: [PATCH 15/43] Use local and remapped paths where appropriate --- src/common.rs | 4 +++- src/debuginfo/line_info.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common.rs b/src/common.rs index 92e4435565ee7..c12d6d0f14143 100644 --- a/src/common.rs +++ b/src/common.rs @@ -334,7 +334,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span); let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo()); let const_loc = self.tcx.const_caller_location(( - rustc_span::symbol::Symbol::intern(&caller.file.name.to_string()), + rustc_span::symbol::Symbol::intern( + &caller.file.name.prefer_remapped().to_string_lossy(), + ), caller.line as u32, caller.col_display as u32 + 1, )); diff --git a/src/debuginfo/line_info.rs b/src/debuginfo/line_info.rs index b3f700bc46794..9eb067706309e 100644 --- a/src/debuginfo/line_info.rs +++ b/src/debuginfo/line_info.rs @@ -87,7 +87,7 @@ fn line_program_add_file( filename => { let dir_id = line_program.default_directory(); let dummy_file_name = LineString::new( - filename.to_string().into_bytes(), + filename.prefer_remapped().to_string().into_bytes(), line_program.encoding(), line_strings, ); From 91dc6967fe5ebb8ff773a168e549482c18bb92d8 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Wed, 24 Mar 2021 21:45:09 -0700 Subject: [PATCH 16/43] Implement RFC 2951: Native link modifiers This commit implements both the native linking modifiers infrastructure as well as an initial attempt at the individual modifiers from the RFC. It also introduces a feature flag for the general syntax along with individual feature flags for each modifier. --- src/archive.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 7583fc424071e..fc0823302e018 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -85,8 +85,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { )); } - fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) { - let location = find_library(name, &self.lib_search_paths, self.sess); + fn add_native_library(&mut self, name: rustc_span::symbol::Symbol, verbatim: bool) { + let location = find_library(name, verbatim, &self.lib_search_paths, self.sess); self.add_archive(location.clone(), |_| false).unwrap_or_else(|e| { panic!("failed to add native library {}: {}", location.to_string_lossy(), e); }); From 8794d07081671b207f725518c2b632eee5c2e326 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 7 May 2021 12:58:54 +0200 Subject: [PATCH 17/43] Avoid trailing / in git path for Wasmtime Git seems to not be able to handle it in some cases. Thanks to @vramana for reporting this on Zulip. --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 14 +++++++------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09bb37292ef3d..a5708d42b3466 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-entity", ] @@ -42,7 +42,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-bforest", "cranelift-codegen-meta", @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -67,17 +67,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" [[package]] name = "cranelift-entity" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" [[package]] name = "cranelift-frontend" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-codegen", "log", @@ -88,7 +88,7 @@ dependencies = [ [[package]] name = "cranelift-jit" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "anyhow", "cranelift-codegen", @@ -106,7 +106,7 @@ dependencies = [ [[package]] name = "cranelift-module" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "anyhow", "cranelift-codegen", @@ -117,7 +117,7 @@ dependencies = [ [[package]] name = "cranelift-native" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -126,7 +126,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" dependencies = [ "anyhow", "cranelift-codegen", diff --git a/Cargo.toml b/Cargo.toml index ff1433d1cb632..b9dac0890a862 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,12 +9,12 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } -cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } -cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } +cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main", features = ["unwind"] } +cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } +cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } +cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } +cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main", optional = true } +cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } target-lexicon = "0.12.0" gimli = { version = "0.23.0", default-features = false, features = ["write"]} object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } @@ -25,7 +25,7 @@ libloading = { version = "0.6.0", optional = true } smallvec = "1.6.1" # Uncomment to use local checkout of cranelift -#[patch."https://github.com/bytecodealliance/wasmtime/"] +#[patch."https://github.com/bytecodealliance/wasmtime.git"] #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } #cranelift-module = { path = "../wasmtime/cranelift/module" } From 62f64af43051fde2b9e643a0f5f06cf88e602a59 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 29 Mar 2021 11:39:13 +0200 Subject: [PATCH 18/43] Use the object crate for metadata reading --- src/lib.rs | 2 +- src/metadata.rs | 66 +------------------------------------------------ 2 files changed, 2 insertions(+), 66 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 32f403957025a..1ade7c77d123a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -165,7 +165,7 @@ impl CodegenBackend for CraneliftCodegenBackend { } fn metadata_loader(&self) -> Box { - Box::new(crate::metadata::CraneliftMetadataLoader) + Box::new(rustc_codegen_ssa::back::metadata::DefaultMetadataLoader) } fn provide(&self, _providers: &mut Providers) {} diff --git a/src/metadata.rs b/src/metadata.rs index 882232fde09d2..ab238244d68d5 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -1,73 +1,9 @@ -//! Reading and writing of the rustc metadata for rlibs and dylibs +//! Writing of the rustc metadata for dylibs -use std::fs::File; -use std::path::Path; - -use rustc_codegen_ssa::METADATA_FILENAME; -use rustc_data_structures::memmap::Mmap; -use rustc_data_structures::owning_ref::OwningRef; -use rustc_data_structures::rustc_erase_owner; -use rustc_data_structures::sync::MetadataRef; -use rustc_middle::middle::cstore::MetadataLoader; use rustc_middle::ty::TyCtxt; -use rustc_target::spec::Target; use crate::backend::WriteMetadata; -/// The metadata loader used by cg_clif. -/// -/// The metadata is stored in the same format as cg_llvm. -/// -/// # Metadata location -/// -///
-///
rlib
-///
The metadata can be found in the `lib.rmeta` file inside of the ar archive.
-///
dylib
-///
The metadata can be found in the `.rustc` section of the shared library.
-///
-pub(crate) struct CraneliftMetadataLoader; - -fn load_metadata_with( - path: &Path, - f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, -) -> Result { - let file = File::open(path).map_err(|e| format!("{:?}", e))?; - let data = unsafe { Mmap::map(file) }.map_err(|e| format!("{:?}", e))?; - let metadata = OwningRef::new(data).try_map(f)?; - return Ok(rustc_erase_owner!(metadata.map_owner_box())); -} - -impl MetadataLoader for CraneliftMetadataLoader { - fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result { - load_metadata_with(path, |data| { - let archive = object::read::archive::ArchiveFile::parse(&*data) - .map_err(|e| format!("{:?}", e))?; - - for entry_result in archive.members() { - let entry = entry_result.map_err(|e| format!("{:?}", e))?; - if entry.name() == METADATA_FILENAME.as_bytes() { - return Ok(entry.data()); - } - } - - Err("couldn't find metadata entry".to_string()) - }) - } - - fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result { - use object::{Object, ObjectSection}; - - load_metadata_with(path, |data| { - let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?; - file.section_by_name(".rustc") - .ok_or("no .rustc section")? - .data() - .map_err(|e| format!("failed to read .rustc section: {:?}", e)) - }) - } -} - // Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112 pub(crate) fn write_metadata(tcx: TyCtxt<'_>, object: &mut O) { use snap::write::FrameEncoder; From bcc68c298a4e8c323151355404cdc1e87f22436b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 9 May 2021 13:17:02 +0200 Subject: [PATCH 19/43] Rustup to rustc 1.54.0-nightly (881c1ac40 2021-05-08) --- example/std_example.rs | 16 +-------- rust-toolchain | 2 +- src/constant.rs | 79 +++++++++++++++++++++++++++++++++++++++++- src/inline_asm.rs | 30 +++++++++++----- 4 files changed, 101 insertions(+), 26 deletions(-) diff --git a/example/std_example.rs b/example/std_example.rs index 77ba72df8ef37..7d608df9253df 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -187,20 +187,6 @@ unsafe fn test_mm_slli_si128() { ); let r = _mm_slli_si128(a, 16); assert_eq_m128i(r, _mm_set1_epi8(0)); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_slli_si128(a, -1); - assert_eq_m128i(_mm_set1_epi8(0), r); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_slli_si128(a, -0x80000000); - assert_eq_m128i(r, _mm_set1_epi8(0)); } #[cfg(target_arch = "x86_64")] @@ -295,7 +281,7 @@ unsafe fn test_mm_extract_epi8() { 8, 9, 10, 11, 12, 13, 14, 15 ); let r1 = _mm_extract_epi8(a, 0); - let r2 = _mm_extract_epi8(a, 19); + let r2 = _mm_extract_epi8(a, 3); assert_eq!(r1, 0xFF); assert_eq!(r2, 3); } diff --git a/rust-toolchain b/rust-toolchain index 70543cb6c6637..e5789427afc68 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-05-04" +channel = "nightly-2021-05-09" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/src/constant.rs b/src/constant.rs index 39daf887a22fe..7728a3324533f 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -448,12 +448,89 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( operand: &Operand<'tcx>, ) -> Option> { match operand { - Operand::Copy(_) | Operand::Move(_) => None, Operand::Constant(const_) => match const_.literal { ConstantKind::Ty(const_) => { fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val.try_to_value() } ConstantKind::Val(val, _) => Some(val), }, + // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored + // inside a temporary before being passed to the intrinsic requiring the const argument. + // This code tries to find a single constant defining definition of the referenced local. + Operand::Copy(place) | Operand::Move(place) => { + if !place.projection.is_empty() { + return None; + } + let mut computed_const_val = None; + for bb_data in fx.mir.basic_blocks() { + for stmt in &bb_data.statements { + match &stmt.kind { + StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => { + match &local_and_rvalue.1 { + Rvalue::Cast(CastKind::Misc, operand, ty) => { + if computed_const_val.is_some() { + return None; // local assigned twice + } + if !matches!(ty.kind(), ty::Uint(_) | ty::Int(_)) { + return None; + } + let const_val = mir_operand_get_const_val(fx, operand)?; + if fx.layout_of(ty).size + != const_val.try_to_scalar_int()?.size() + { + return None; + } + computed_const_val = Some(const_val); + } + Rvalue::Use(operand) => { + computed_const_val = mir_operand_get_const_val(fx, operand) + } + _ => return None, + } + } + StatementKind::SetDiscriminant { place: stmt_place, variant_index: _ } + if &**stmt_place == place => + { + return None; + } + StatementKind::LlvmInlineAsm(_) | StatementKind::CopyNonOverlapping(_) => { + return None; + } // conservative handling + StatementKind::Assign(_) + | StatementKind::FakeRead(_) + | StatementKind::SetDiscriminant { .. } + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Retag(_, _) + | StatementKind::AscribeUserType(_, _) + | StatementKind::Coverage(_) + | StatementKind::Nop => {} + } + } + match &bb_data.terminator().kind { + TerminatorKind::Goto { .. } + | TerminatorKind::SwitchInt { .. } + | TerminatorKind::Resume + | TerminatorKind::Abort + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::Drop { .. } + | TerminatorKind::Assert { .. } => {} + TerminatorKind::DropAndReplace { .. } + | TerminatorKind::Yield { .. } + | TerminatorKind::GeneratorDrop + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } => unreachable!(), + TerminatorKind::InlineAsm { .. } => return None, + TerminatorKind::Call { destination: Some((call_place, _)), .. } + if call_place == place => + { + return None; + } + TerminatorKind::Call { .. } => {} + } + } + computed_const_val + } } } diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 4ab4c2957ca4e..09c5e6031c78d 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -24,14 +24,22 @@ pub(crate) fn codegen_inline_asm<'tcx>( let true_ = fx.bcx.ins().iconst(types::I32, 1); fx.bcx.ins().trapnz(true_, TrapCode::User(1)); return; - } else if template[0] == InlineAsmTemplatePiece::String("mov rsi, rbx".to_string()) - && template[1] == InlineAsmTemplatePiece::String("\n".to_string()) - && template[2] == InlineAsmTemplatePiece::String("cpuid".to_string()) - && template[3] == InlineAsmTemplatePiece::String("\n".to_string()) - && template[4] == InlineAsmTemplatePiece::String("xchg rsi, rbx".to_string()) + } else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string()) + && matches!( + template[1], + InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ } + ) + && template[2] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[3] == InlineAsmTemplatePiece::String("cpuid".to_string()) + && template[4] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[5] == InlineAsmTemplatePiece::String("xchgq %rbx, ".to_string()) + && matches!( + template[6], + InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ } + ) { assert_eq!(operands.len(), 4); - let (leaf, eax_place) = match operands[0] { + let (leaf, eax_place) = match operands[1] { InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { let reg = expect_reg(reg); assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax)); @@ -42,10 +50,14 @@ pub(crate) fn codegen_inline_asm<'tcx>( } _ => unreachable!(), }; - let ebx_place = match operands[1] { + let ebx_place = match operands[0] { InlineAsmOperand::Out { reg, late: true, place } => { - let reg = expect_reg(reg); - assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::si)); + assert_eq!( + reg, + InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86( + X86InlineAsmRegClass::reg + )) + ); crate::base::codegen_place(fx, place.unwrap()) } _ => unreachable!(), From 8d969691d43e317f4648ef2c234e5efe9b96e790 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 9 May 2021 13:24:23 +0200 Subject: [PATCH 20/43] Re-enable fixed libcore tests They were fixed in 6d6c5742896fab93788b4dabb6ba2fcf1881f5e2 --- ...022-core-Disable-not-compiling-tests.patch | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/patches/0022-core-Disable-not-compiling-tests.patch b/patches/0022-core-Disable-not-compiling-tests.patch index 8cfffe580a1f0..ba0eaacd82870 100644 --- a/patches/0022-core-Disable-not-compiling-tests.patch +++ b/patches/0022-core-Disable-not-compiling-tests.patch @@ -39,46 +39,6 @@ index a35897e..f0bf645 100644 pub fn decode_finite(v: T) -> Decoded { match decode(v).1 { -diff --git a/library/core/tests/num/int_macros.rs b/library/core/tests/num/int_macros.rs -index 0475aeb..9558198 100644 ---- a/library/core/tests/num/int_macros.rs -+++ b/library/core/tests/num/int_macros.rs -@@ -88,6 +88,7 @@ mod tests { - assert_eq!(x.trailing_ones(), 0); - } - -+ /* - #[test] - fn test_rotate() { - assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A); -@@ -112,6 +113,7 @@ mod tests { - assert_eq!(B.rotate_left(128), B); - assert_eq!(C.rotate_left(128), C); - } -+ */ - - #[test] - fn test_swap_bytes() { -diff --git a/library/core/tests/num/uint_macros.rs b/library/core/tests/num/uint_macros.rs -index 04ed14f..a6e372e 100644 ---- a/library/core/tests/num/uint_macros.rs -+++ b/library/core/tests/num/uint_macros.rs -@@ -52,6 +52,7 @@ mod tests { - assert_eq!(x.trailing_ones(), 0); - } - -+ /* - #[test] - fn test_rotate() { - assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A); -@@ -76,6 +77,7 @@ mod tests { - assert_eq!(B.rotate_left(128), B); - assert_eq!(C.rotate_left(128), C); - } -+ */ - - #[test] - fn test_swap_bytes() { diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs index 1a6be3a..42dbd59 100644 --- a/library/core/tests/ptr.rs From 801eb1914f2b8232dfd2870e7366836c2d88c64b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 9 May 2021 13:26:54 +0200 Subject: [PATCH 21/43] Remove unnecessary compiler builtins patch --- ...ns-Disable-128bit-atomic-operations.patch} | 0 ...builtins-Remove-rotate_left-from-Int.patch | 35 ------------------- 2 files changed, 35 deletions(-) rename crate_patches/{0002-compiler-builtins-Disable-128bit-atomic-operations.patch => 0001-compiler-builtins-Disable-128bit-atomic-operations.patch} (100%) delete mode 100644 crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch diff --git a/crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch b/crate_patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch similarity index 100% rename from crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch rename to crate_patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch diff --git a/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch b/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch deleted file mode 100644 index b4acc4f5b7365..0000000000000 --- a/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 7078cca3cb614e1e82da428380b4e16fc3afef46 Mon Sep 17 00:00:00 2001 -From: bjorn3 -Date: Thu, 21 Jan 2021 14:46:36 +0100 -Subject: [PATCH] Remove rotate_left from Int - ---- - src/int/mod.rs | 5 ----- - 1 file changed, 5 deletions(-) - -diff --git a/src/int/mod.rs b/src/int/mod.rs -index 06054c8..3bea17b 100644 ---- a/src/int/mod.rs -+++ b/src/int/mod.rs -@@ -85,7 +85,6 @@ pub trait Int: - fn wrapping_sub(self, other: Self) -> Self; - fn wrapping_shl(self, other: u32) -> Self; - fn wrapping_shr(self, other: u32) -> Self; -- fn rotate_left(self, other: u32) -> Self; - fn overflowing_add(self, other: Self) -> (Self, bool); - fn leading_zeros(self) -> u32; - } -@@ -209,10 +208,6 @@ macro_rules! int_impl_common { - ::wrapping_shr(self, other) - } - -- fn rotate_left(self, other: u32) -> Self { -- ::rotate_left(self, other) -- } -- - fn overflowing_add(self, other: Self) -> (Self, bool) { - ::overflowing_add(self, other) - } --- -2.26.2.7.g19db9cfb68 - From 459a51906d4ae5f345e2a58fc9c7056179fd7b8a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 10 May 2021 13:13:52 +0200 Subject: [PATCH 22/43] Update Cranelift --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5708d42b3466..e88b990a461fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" dependencies = [ "cranelift-entity", ] @@ -42,7 +42,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" dependencies = [ "cranelift-bforest", "cranelift-codegen-meta", @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -67,17 +67,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" [[package]] name = "cranelift-entity" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" [[package]] name = "cranelift-frontend" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" dependencies = [ "cranelift-codegen", "log", @@ -88,7 +88,7 @@ dependencies = [ [[package]] name = "cranelift-jit" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" dependencies = [ "anyhow", "cranelift-codegen", @@ -106,7 +106,7 @@ dependencies = [ [[package]] name = "cranelift-module" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" dependencies = [ "anyhow", "cranelift-codegen", @@ -117,7 +117,7 @@ dependencies = [ [[package]] name = "cranelift-native" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -126,7 +126,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#fe76c5915980208d1971b491ffed0a139dafb7f9" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" dependencies = [ "anyhow", "cranelift-codegen", From 4663ed7bd9ae04dea5beb6d852a37837f00d4a23 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 11 May 2021 13:22:38 +0200 Subject: [PATCH 23/43] Rustup to rustc 1.54.0-nightly (79e50bf77 2021-05-10) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index e5789427afc68..16df0ed75089e 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-05-09" +channel = "nightly-2021-05-11" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 7c40338ba13aa38c8ba4442f92d50155ad9f6ddd Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 11 May 2021 14:22:05 +0200 Subject: [PATCH 24/43] Implement imported_main feature Fixes #1164 --- src/driver/aot.rs | 8 +++++++- src/driver/jit.rs | 1 + src/main_shim.rs | 9 +++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 9c5cd53d8669d..99c15a26fe53f 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -134,7 +134,13 @@ fn module_codegen( } } } - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context, false); + crate::main_shim::maybe_create_entry_wrapper( + tcx, + &mut module, + &mut cx.unwind_context, + false, + cgu.is_primary(), + ); let debug_context = cx.debug_context; let unwind_context = cx.unwind_context; diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 526caeb4f03f5..fac48401a7eed 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -45,6 +45,7 @@ fn create_jit_module<'tcx>( &mut jit_module, &mut cx.unwind_context, true, + true, ); (jit_module, cx) diff --git a/src/main_shim.rs b/src/main_shim.rs index d504024a3358e..f231e791abddd 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -14,6 +14,7 @@ pub(crate) fn maybe_create_entry_wrapper( module: &mut impl Module, unwind_context: &mut UnwindContext, is_jit: bool, + is_primary_cgu: bool, ) { let (main_def_id, is_main_fn) = match tcx.entry_fn(LOCAL_CRATE) { Some((def_id, entry_ty)) => ( @@ -26,8 +27,12 @@ pub(crate) fn maybe_create_entry_wrapper( None => return, }; - let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() { + if main_def_id.is_local() { + let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); + if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() { + return; + } + } else if !is_primary_cgu { return; } From 24c459c2e51fecb6f9e1453bb1d7ab819ff471a3 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 11 May 2021 14:25:56 +0200 Subject: [PATCH 25/43] Use declare_anonymous_data for anonymous_str --- src/base.rs | 2 +- src/common.rs | 10 ++-------- src/trap.rs | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/base.rs b/src/base.rs index 3ec5c14ff17a2..2e16dc37c08d6 100644 --- a/src/base.rs +++ b/src/base.rs @@ -870,7 +870,7 @@ pub(crate) fn codegen_operand<'tcx>( pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) { let location = fx.get_caller_location(span).load_scalar(fx); - let msg_ptr = fx.anonymous_str("assert", msg_str); + let msg_ptr = fx.anonymous_str(msg_str); let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); let args = [msg_ptr, msg_len, location]; diff --git a/src/common.rs b/src/common.rs index 92e4435565ee7..ed6c4411a3d24 100644 --- a/src/common.rs +++ b/src/common.rs @@ -345,18 +345,12 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { self.module.isa().triple() } - pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - msg.hash(&mut hasher); - let msg_hash = hasher.finish(); + pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value { let mut data_ctx = DataContext::new(); data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice()); let msg_id = self .module - .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false) + .declare_anonymous_data(false, false) .unwrap(); // Ignore DuplicateDefinition error, as the data will be the same diff --git a/src/trap.rs b/src/trap.rs index 819c8b51558a0..21d3e68dbc792 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -21,7 +21,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { } let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg); - let msg_ptr = fx.anonymous_str("trap", &real_msg); + let msg_ptr = fx.anonymous_str(&real_msg); fx.bcx.ins().call(puts, &[msg_ptr]); } From a606fdec730da2eb6f29230739cedb70ec3a7e0e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 11 May 2021 14:26:32 +0200 Subject: [PATCH 26/43] Fix syntax in patch section of Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b9dac0890a862..cdc3693de9787 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ smallvec = "1.6.1" #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } #cranelift-module = { path = "../wasmtime/cranelift/module" } -#cranelift-native = { path = ../wasmtime/cranelift/native" } +#cranelift-native = { path = "../wasmtime/cranelift/native" } #cranelift-jit = { path = "../wasmtime/cranelift/jit" } #cranelift-object = { path = "../wasmtime/cranelift/object" } From f22ab98d4c2dee3e0d569e06d99dbfdaa6cc52ae Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 11 May 2021 11:26:52 +0200 Subject: [PATCH 27/43] Use () in dependency_formats. --- src/allocator.rs | 2 +- src/driver/jit.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allocator.rs b/src/allocator.rs index a09e32577869e..357a9f2daf746 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -13,7 +13,7 @@ pub(crate) fn codegen( module: &mut impl Module, unwind_context: &mut UnwindContext, ) -> bool { - let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| { + let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; list.iter().any(|&linkage| linkage == Linkage::Dynamic) }); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 53c93f6a9ddca..bf601c70c094f 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -179,7 +179,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { let mut dylib_paths = Vec::new(); let crate_info = CrateInfo::new(tcx); - let formats = tcx.dependency_formats(LOCAL_CRATE); + let formats = tcx.dependency_formats(()); let data = &formats .iter() .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable) From 3a869ca88c847c8e004a3d4f74247c8378cfa11a Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 11 May 2021 12:00:59 +0200 Subject: [PATCH 28/43] Use () for entry_fn. --- src/main_shim.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main_shim.rs b/src/main_shim.rs index d504024a3358e..d1958c5f96b86 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -15,7 +15,7 @@ pub(crate) fn maybe_create_entry_wrapper( unwind_context: &mut UnwindContext, is_jit: bool, ) { - let (main_def_id, is_main_fn) = match tcx.entry_fn(LOCAL_CRATE) { + let (main_def_id, is_main_fn) = match tcx.entry_fn(()) { Some((def_id, entry_ty)) => ( def_id, match entry_ty { From 915d7663076ffc8a0633ed2ee47ce80e4dcdac19 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 11 May 2021 14:39:04 +0200 Subject: [PATCH 29/43] Use () for codegen queries. --- src/driver/aot.rs | 10 +++++----- src/driver/jit.rs | 3 +-- src/pretty_clif.rs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index e5f06551bb643..e776cd0c620af 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -41,7 +41,7 @@ fn emit_module( unwind_context.emit(&mut product); - let tmp_file = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&name)); + let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); let obj = product.object.write().unwrap(); if let Err(err) = std::fs::write(&tmp_file, obj) { tcx.sess.fatal(&format!("error writing object file: {}", err)); @@ -73,7 +73,7 @@ fn reuse_workproduct_for_cgu( let work_product = cgu.work_product(tcx); if let Some(saved_file) = &work_product.saved_file { let obj_out = tcx - .output_filenames(LOCAL_CRATE) + .output_filenames(()) .temp_path(OutputType::Object, Some(&cgu.name().as_str())); object = Some(obj_out.clone()); let source_file = rustc_incremental::in_incr_comp_dir(&incr_comp_session_dir, &saved_file); @@ -179,7 +179,7 @@ pub(crate) fn run_aot( let mut work_products = FxHashMap::default(); let cgus = if tcx.sess.opts.output_types.should_codegen() { - tcx.collect_and_partition_mono_items(LOCAL_CRATE).1 + tcx.collect_and_partition_mono_items(()).1 } else { // If only `--emit metadata` is used, we shouldn't perform any codegen. // Also `tcx.collect_and_partition_mono_items` may panic in that case. @@ -265,7 +265,7 @@ pub(crate) fn run_aot( .to_string(); let tmp_file = tcx - .output_filenames(LOCAL_CRATE) + .output_filenames(()) .temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); let obj = crate::backend::with_object(tcx.sess, &metadata_cgu_name, |object| { @@ -342,7 +342,7 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { .join("\n"); let output_object_file = - tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(cgu_name)); + tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); // Assemble `global_asm` let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index bf601c70c094f..632e86da736ab 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -8,7 +8,6 @@ use std::os::raw::{c_char, c_int}; use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; -use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; @@ -66,7 +65,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { matches!(backend_config.codegen_mode, CodegenMode::JitLazy), ); - let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let (_, cgus) = tcx.collect_and_partition_mono_items(()); let mono_items = cgus .iter() .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 158811c5eaf49..cd8c5b5160836 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -214,7 +214,7 @@ pub(crate) fn write_ir_file( return; } - let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif"); + let clif_output_dir = tcx.output_filenames(()).with_extension("clif"); match std::fs::create_dir(&clif_output_dir) { Ok(()) => {} From d342a3490b6f85f5bec80a7b90a576e5044719aa Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sun, 11 Apr 2021 20:51:28 +0100 Subject: [PATCH 30/43] Add support for const operands and options to global_asm! On x86, the default syntax is also switched to Intel to match asm! --- src/driver/aot.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index e5f06551bb643..004e6bddaf3ee 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::back::linker::LinkerInfo; use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -125,9 +126,19 @@ fn module_codegen( MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { let item = cx.tcx.hir().item(item_id); - if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { - cx.global_asm.push_str(&*asm.as_str()); - cx.global_asm.push_str("\n\n"); + if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { + if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { + cx.global_asm.push_str("\n.intel_syntax noprefix\n"); + } else { + cx.global_asm.push_str("\n.att_syntax\n"); + } + for piece in asm.template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s), + InlineAsmTemplatePiece::Placeholder { .. } => todo!(), + } + } + cx.global_asm.push_str("\n.att_syntax\n\n"); } else { bug!("Expected GlobalAsm found {:?}", item); } From 328f06718a71e786e463ae6a5d1874bd13061cde Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 16 May 2021 10:45:18 +0200 Subject: [PATCH 31/43] Update Cranelift --- Cargo.lock | 48 ++++++++++-------------------------------------- 1 file changed, 10 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e88b990a461fd..f83d9846f46f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" dependencies = [ "cranelift-entity", ] @@ -42,7 +42,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" dependencies = [ "cranelift-bforest", "cranelift-codegen-meta", @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -67,17 +67,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" [[package]] name = "cranelift-entity" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" [[package]] name = "cranelift-frontend" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" dependencies = [ "cranelift-codegen", "log", @@ -88,14 +88,13 @@ dependencies = [ [[package]] name = "cranelift-jit" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" dependencies = [ "anyhow", "cranelift-codegen", "cranelift-entity", "cranelift-module", "cranelift-native", - "errno", "libc", "log", "region", @@ -106,7 +105,7 @@ dependencies = [ [[package]] name = "cranelift-module" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" dependencies = [ "anyhow", "cranelift-codegen", @@ -117,7 +116,7 @@ dependencies = [ [[package]] name = "cranelift-native" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -126,7 +125,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#f60ae73d151909ef64964524f9dc927eeee0b317" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" dependencies = [ "anyhow", "cranelift-codegen", @@ -145,33 +144,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "errno" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067" -dependencies = [ - "gcc", - "libc", -] - -[[package]] -name = "gcc" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" - [[package]] name = "gimli" version = "0.23.0" From a9d033d0e5e29551fc578e8ea4eb3d5f43166b94 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 16 May 2021 10:46:34 +0200 Subject: [PATCH 32/43] Return 0 from cpuid for all subleaves of leaf 7 This is necessary to support the cpufeatures crate Fixes #1173 --- src/intrinsics/cpuid.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/intrinsics/cpuid.rs b/src/intrinsics/cpuid.rs index 9de12e759bcc8..d02dfd93c3ee3 100644 --- a/src/intrinsics/cpuid.rs +++ b/src/intrinsics/cpuid.rs @@ -12,6 +12,7 @@ pub(crate) fn codegen_cpuid_call<'tcx>( ) -> (Value, Value, Value, Value) { let leaf_0 = fx.bcx.create_block(); let leaf_1 = fx.bcx.create_block(); + let leaf_7 = fx.bcx.create_block(); let leaf_8000_0000 = fx.bcx.create_block(); let leaf_8000_0001 = fx.bcx.create_block(); let unsupported_leaf = fx.bcx.create_block(); @@ -25,6 +26,7 @@ pub(crate) fn codegen_cpuid_call<'tcx>( let mut switch = cranelift_frontend::Switch::new(); switch.set_entry(0, leaf_0); switch.set_entry(1, leaf_1); + switch.set_entry(7, leaf_7); switch.set_entry(0x8000_0000, leaf_8000_0000); switch.set_entry(0x8000_0001, leaf_8000_0001); switch.emit(&mut fx.bcx, leaf, unsupported_leaf); @@ -43,6 +45,11 @@ pub(crate) fn codegen_cpuid_call<'tcx>( let edx_features = fx.bcx.ins().iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */); fx.bcx.ins().jump(dest, &[cpu_signature, additional_information, ecx_features, edx_features]); + fx.bcx.switch_to_block(leaf_7); + // This leaf technically has subleaves, but we just return zero for all subleaves. + let zero = fx.bcx.ins().iconst(types::I32, 0); + fx.bcx.ins().jump(dest, &[zero, zero, zero, zero]); + fx.bcx.switch_to_block(leaf_8000_0000); let extended_max_basic_leaf = fx.bcx.ins().iconst(types::I32, 0); let zero = fx.bcx.ins().iconst(types::I32, 0); From 5e52b61040223ce8b107fb759f7981d5fd90bdcc Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 16 May 2021 18:53:20 +0200 Subject: [PATCH 33/43] CTFE core engine allocation & memory API improvemenets - make Allocation API offset-based (no more Pointer) - make Memory API higher-level (combine checking for access and getting access into one operation) --- src/constant.rs | 5 ++--- src/intrinsics/simd.rs | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/constant.rs b/src/constant.rs index 0a0e02d26394e..c0f2920652f61 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -6,7 +6,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::ErrorReported; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::interpret::{ - read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Pointer, Scalar, + alloc_range, read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar, }; use rustc_middle::ty::ConstKind; @@ -176,8 +176,7 @@ pub(crate) fn codegen_const_value<'tcx>( std::iter::repeat(0).take(size.bytes_usize()).collect::>(), align, ); - let ptr = Pointer::new(AllocId(!0), Size::ZERO); // The alloc id is never used - alloc.write_scalar(fx, ptr, x.into(), size).unwrap(); + alloc.write_scalar(fx, alloc_range(Size::ZERO, size), x.into()).unwrap(); let alloc = fx.tcx.intern_const_alloc(alloc); return CValue::by_ref(pointer_for_allocation(fx, alloc), layout); } diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index 940d2514f7446..c2f469fa021e1 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -86,9 +86,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let idx_bytes = match idx_const { ConstValue::ByRef { alloc, offset } => { - let ptr = Pointer::new(AllocId(0 /* dummy */), offset); let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */); - alloc.get_bytes(fx, ptr, size).unwrap() + alloc.get_bytes(fx, alloc_range(offset, size)).unwrap() } _ => unreachable!("{:?}", idx_const), }; From beab7510cfbfb3aaa541922f2298b91b5bfff99d Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 18 May 2021 19:41:04 +0200 Subject: [PATCH 34/43] Move some code around a bit --- src/base.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/base.rs b/src/base.rs index 2e16dc37c08d6..ec3e17e5b758d 100644 --- a/src/base.rs +++ b/src/base.rs @@ -110,11 +110,6 @@ pub(crate) fn codegen_fn<'tcx>( // Verify function verify_func(tcx, &clif_comments, &context.func); - // Perform rust specific optimizations - tcx.sess.time("optimize clif ir", || { - crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments); - }); - // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` // instruction, which doesn't have an encoding. context.compute_cfg(); @@ -125,10 +120,14 @@ pub(crate) fn codegen_fn<'tcx>( // invalidate it when it would change. context.domtree.clear(); - context.want_disasm = crate::pretty_clif::should_write_ir(tcx); + // Perform rust specific optimizations + tcx.sess.time("optimize clif ir", || { + crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments); + }); // Define function tcx.sess.time("define function", || { + context.want_disasm = crate::pretty_clif::should_write_ir(tcx); module .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {}) .unwrap() From 6e06f22bb6f940fd899247c5985e1be6f6551e48 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 18 May 2021 19:51:31 +0200 Subject: [PATCH 35/43] Rustup to rustc 1.54.0-nightly (3e99439f4 2021-05-17) --- build_sysroot/Cargo.lock | 2 +- build_sysroot/prepare_sysroot_src.sh | 2 +- rust-toolchain | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index dd5228d3ba4ca..201b74f7f76e7 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -56,7 +56,7 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.42" +version = "0.1.43" dependencies = [ "rustc-std-workspace-core", ] diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh index 2b0feb93d4dea..54b7a94750c52 100755 --- a/build_sysroot/prepare_sysroot_src.sh +++ b/build_sysroot/prepare_sysroot_src.sh @@ -32,7 +32,7 @@ popd git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned" pushd compiler-builtins git checkout -- . -git checkout 0.1.42 +git checkout 0.1.43 git apply ../../crate_patches/000*-compiler-builtins-*.patch popd diff --git a/rust-toolchain b/rust-toolchain index 16df0ed75089e..fea0dba33f763 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-05-11" +channel = "nightly-2021-05-18" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 9d41e8f90dc837cff4960a67d839db015a15ed74 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 23 May 2021 12:44:05 +0200 Subject: [PATCH 36/43] (try to) fix cranelift --- src/constant.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/constant.rs b/src/constant.rs index c0f2920652f61..6b132e4ff0fb8 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -2,6 +2,7 @@ use rustc_span::DUMMY_SP; +use rustc_ast::Mutability; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::ErrorReported; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; @@ -175,6 +176,7 @@ pub(crate) fn codegen_const_value<'tcx>( let mut alloc = Allocation::from_bytes( std::iter::repeat(0).take(size.bytes_usize()).collect::>(), align, + Mutability::Not, ); alloc.write_scalar(fx, alloc_range(Size::ZERO, size), x.into()).unwrap(); let alloc = fx.tcx.intern_const_alloc(alloc); From 2f9ca10b1ddb8e5f25ee2dec1e1be6521612af5a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 24 May 2021 16:06:54 +0200 Subject: [PATCH 37/43] Update Cranelift, object and gimli --- Cargo.lock | 48 +++++++++++++++++++++++------------------------ Cargo.toml | 4 ++-- src/archive.rs | 2 +- src/driver/jit.rs | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f83d9846f46f7..a6f5925149b92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,16 +33,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-bforest", "cranelift-codegen-meta", @@ -57,8 +57,8 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -66,18 +66,18 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" [[package]] name = "cranelift-entity" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" [[package]] name = "cranelift-frontend" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-codegen", "log", @@ -87,8 +87,8 @@ dependencies = [ [[package]] name = "cranelift-jit" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "anyhow", "cranelift-codegen", @@ -104,8 +104,8 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "anyhow", "cranelift-codegen", @@ -115,8 +115,8 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -124,8 +124,8 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.73.0" -source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#64202cc0171224889eac6e747f5456cb8c8af7fe" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "anyhow", "cranelift-codegen", @@ -146,9 +146,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" dependencies = [ "indexmap", ] @@ -205,9 +205,9 @@ dependencies = [ [[package]] name = "object" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" +checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170" dependencies = [ "crc32fast", "indexmap", diff --git a/Cargo.toml b/Cargo.toml index cdc3693de9787..fd149af454735 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", b cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main", optional = true } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } target-lexicon = "0.12.0" -gimli = { version = "0.23.0", default-features = false, features = ["write"]} -object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } +gimli = { version = "0.24.0", default-features = false, features = ["write"]} +object = { version = "0.24.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" } indexmap = "1.0.2" diff --git a/src/archive.rs b/src/archive.rs index fc0823302e018..bd54adc53ee2a 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -160,7 +160,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { }; if !self.no_builtin_ranlib { - match object::File::parse(&data) { + match object::File::parse(&*data) { Ok(object) => { symbol_table.insert( entry_name.as_bytes().to_vec(), diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 199327a9ff1ce..4a99cb727c830 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -207,7 +207,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { use object::{Object, ObjectSymbol}; let lib = libloading::Library::new(&path).unwrap(); let obj = std::fs::read(path).unwrap(); - let obj = object::File::parse(&obj).unwrap(); + let obj = object::File::parse(&*obj).unwrap(); imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| { let name = symbol.name().unwrap().to_string(); if name.is_empty() || !symbol.is_global() || symbol.is_undefined() { From 9fc56fb2f47d0b8a825d1117bdae0ca80d0d3304 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 24 May 2021 17:16:45 +0200 Subject: [PATCH 38/43] Update rustc bootstrap patch --- scripts/setup_rust_fork.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index e342963de15d1..43c4887669cf6 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -32,11 +32,12 @@ index d95b5b7f17f..00b6f0e3635 100644 [dependencies] core = { path = "../core" } --compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] } -+compiler_builtins = { version = "0.1.42", features = ['rustc-dep-of-std', 'no-asm'] } +-compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std'] } ++compiler_builtins = { version = "0.1.43", features = ['rustc-dep-of-std', 'no-asm'] } [dev-dependencies] rand = "0.7" + rand_xorshift = "0.2" EOF cat > config.toml < Date: Mon, 24 May 2021 18:41:45 +0200 Subject: [PATCH 39/43] Rustfmt --- src/common.rs | 5 +---- src/driver/aot.rs | 13 +++++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/common.rs b/src/common.rs index 8d323efee69c4..488ff6e134956 100644 --- a/src/common.rs +++ b/src/common.rs @@ -350,10 +350,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value { let mut data_ctx = DataContext::new(); data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice()); - let msg_id = self - .module - .declare_anonymous_data(false, false) - .unwrap(); + let msg_id = self.module.declare_anonymous_data(false, false).unwrap(); // Ignore DuplicateDefinition error, as the data will be the same let _ = self.module.define_data(msg_id, &data_ctx); diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 0a104947eae0a..9cf51d15c8ca0 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -73,9 +73,8 @@ fn reuse_workproduct_for_cgu( let mut object = None; let work_product = cgu.work_product(tcx); if let Some(saved_file) = &work_product.saved_file { - let obj_out = tcx - .output_filenames(()) - .temp_path(OutputType::Object, Some(&cgu.name().as_str())); + let obj_out = + tcx.output_filenames(()).temp_path(OutputType::Object, Some(&cgu.name().as_str())); object = Some(obj_out.clone()); let source_file = rustc_incremental::in_incr_comp_dir(&incr_comp_session_dir, &saved_file); if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) { @@ -281,9 +280,8 @@ pub(crate) fn run_aot( .as_str() .to_string(); - let tmp_file = tcx - .output_filenames(()) - .temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); + let tmp_file = + tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); let obj = crate::backend::with_object(tcx.sess, &metadata_cgu_name, |object| { crate::metadata::write_metadata(tcx, object); @@ -358,8 +356,7 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { .collect::>() .join("\n"); - let output_object_file = - tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); + let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); // Assemble `global_asm` let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); From 39ffd9ae4aec7d908dabc9a7b1f384956e197978 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 26 May 2021 14:00:58 +0200 Subject: [PATCH 40/43] Allow switching the regalloc algorithm from the commandline --- src/config.rs | 9 +++++++++ src/lib.rs | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/config.rs b/src/config.rs index e59a0cb0a2323..eef3c8c8d6e2b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -48,6 +48,12 @@ pub struct BackendConfig { /// Can be set using `-Cllvm-args=display_cg_time=...`. pub display_cg_time: bool, + /// The register allocator to use. + /// + /// Defaults to the value of `CG_CLIF_REGALLOC` or `backtracking` otherwise. Can be set using + /// `-Cllvm-args=regalloc=...`. + pub regalloc: String, + /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run /// once before passing the clif ir to Cranelift for compilation. /// @@ -74,6 +80,8 @@ impl Default for BackendConfig { args.split(' ').map(|arg| arg.to_string()).collect() }, display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"), + regalloc: std::env::var("CG_CLIF_REGALLOC") + .unwrap_or_else(|_| "backtracking".to_string()), enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"), disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"), } @@ -93,6 +101,7 @@ impl BackendConfig { match name { "mode" => config.codegen_mode = value.parse()?, "display_cg_time" => config.display_cg_time = parse_bool(name, value)?, + "regalloc" => config.regalloc = value.to_string(), "enable_verifier" => config.enable_verifier = parse_bool(name, value)?, "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?, _ => return Err(format!("Unknown option `{}`", name)), diff --git a/src/lib.rs b/src/lib.rs index a9475997f7553..4ee887cd5afae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -256,6 +256,8 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box { From 56facc424ad2cb4425f3c572901f196895f4d052 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 27 May 2021 12:39:40 +0200 Subject: [PATCH 41/43] Rustup to rustc 1.54.0-nightly (ff2c947c0 2021-05-25) --- build_sysroot/Cargo.lock | 8 ++++---- rust-toolchain | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index 201b74f7f76e7..923deb9aec4c0 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -40,9 +40,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "cc" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" [[package]] name = "cfg-if" @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.94" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" dependencies = [ "rustc-std-workspace-core", ] diff --git a/rust-toolchain b/rust-toolchain index fea0dba33f763..9fe6e093a7b81 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-05-18" +channel = "nightly-2021-05-26" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 40dd3e2b7089b5e96714e064b731f6dbf17c61a9 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 27 May 2021 12:40:08 +0200 Subject: [PATCH 42/43] Rustfmt --- src/constant.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/constant.rs b/src/constant.rs index 1e89f5220b463..3ba12c4e96d68 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -7,7 +7,8 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::ErrorReported; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::interpret::{ - alloc_range, read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar, + alloc_range, read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, + Scalar, }; use rustc_middle::ty::ConstKind; From 6487127a05ef01c7213504c6de98b4f95b9e8191 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 27 May 2021 13:23:32 +0200 Subject: [PATCH 43/43] Remove unused tidy dep exceptions --- src/tools/tidy/src/deps.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index bba689d07a42e..06cec1964a080 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -235,7 +235,6 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ "ar", "autocfg", "bitflags", - "byteorder", "cfg-if", "cranelift-bforest", "cranelift-codegen", @@ -248,9 +247,6 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ "cranelift-native", "cranelift-object", "crc32fast", - "errno", - "errno-dragonfly", - "gcc", "gimli", "hashbrown", "indexmap", @@ -259,17 +255,11 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ "log", "mach", "object", - "proc-macro2", - "quote", "regalloc", "region", "rustc-hash", "smallvec", - "syn", "target-lexicon", - "thiserror", - "thiserror-impl", - "unicode-xid", "winapi", "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu",