Sync rustc_codegen_cranelift

compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml

@@ -0,0 +1,64 @@
+name: Abi-cafe
+
+on:
+  - push
+
+jobs:
+  abi_cafe:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.env.TARGET_TRIPLE }}
+      cancel-in-progress: true
+
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-unknown-linux-gnu
+          - os: macos-latest
+            env:
+              TARGET_TRIPLE: x86_64-apple-darwin
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-msvc
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: build/cg_clif
+        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Set MinGW as the default toolchain
+      if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: rustup set default-host x86_64-pc-windows-gnu
+
+    - name: Use sparse cargo registry
+      run: |
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
+    - name: Prepare dependencies
+      run: ./y.rs prepare
+
+    - name: Build
+      run: ./y.rs build --sysroot none
+
+    - name: Test abi-cafe
+      env:
+        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
+      run: ./y.rs abi-cafe

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
         rustfmt --check build_system/mod.rs
 
 
-  build:
+  test:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
 
@@ -114,63 +114,6 @@ jobs:
       run: ./y.rs test
 
 
-  abi_cafe:
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
-
-    defaults:
-      run:
-        shell: bash
-
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - os: ubuntu-latest
-            env:
-              TARGET_TRIPLE: x86_64-unknown-linux-gnu
-          - os: macos-latest
-            env:
-              TARGET_TRIPLE: x86_64-apple-darwin
-          - os: windows-latest
-            env:
-              TARGET_TRIPLE: x86_64-pc-windows-msvc
-          - os: windows-latest
-            env:
-              TARGET_TRIPLE: x86_64-pc-windows-gnu
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Cache cargo target dir
-      uses: actions/cache@v3
-      with:
-        path: build/cg_clif
-        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
-
-    - name: Set MinGW as the default toolchain
-      if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
-      run: rustup set default-host x86_64-pc-windows-gnu
-
-    - name: Use sparse cargo registry
-      run: |
-        cat >> ~/.cargo/config.toml <<EOF
-        [unstable]
-        sparse-registry = true
-        EOF
-
-    - name: Prepare dependencies
-      run: ./y.rs prepare
-
-    - name: Build
-      run: ./y.rs build --sysroot none
-
-    - name: Test abi-cafe
-      env:
-        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
-      run: ./y.rs abi-cafe
-
-
   bench:
     runs-on: ubuntu-latest
     timeout-minutes: 60

compiler/rustc_codegen_cranelift/.gitignore

@@ -14,3 +14,4 @@ perf.data.old
 /dist
 /rust
 /download
+/git-fixed-subtree.sh

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -57,18 +57,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f3d54eab028f5805ae3b26fd60eca3f3a9cfb76b989d9bab173be3f61356cc3"
+checksum = "a7379abaacee0f14abf3204a7606118f0465785252169d186337bcb75030815a"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2be1d5f2c3cca1efb691844bc1988b89c77291f13f778499a3f3c0cf49c0ed61"
+checksum = "9489fa336927df749631f1008007ced2871068544f40a202ce6d93fbf2366a7b"
 dependencies = [
  "arrayvec",
  "bumpalo",
@@ -87,30 +87,30 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9b1b1089750ce4005893af7ee00bb08a2cf1c9779999c0f7164cbc8ad2e0d2"
+checksum = "05bbb67da91ec721ed57cef2f7c5ef7728e1cd9bde9ffd3ef8601022e73e3239"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc5fbaec51de47297fd7304986fd53c8c0030abbe69728a60d72e1c63559318d"
+checksum = "418ecb2f36032f6665dc1a5e2060a143dbab41d83b784882e97710e890a7a16d"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dab984c94593f876090fae92e984bdcc74d9b1acf740ab5f79036001c65cba13"
+checksum = "7cf583f7b093f291005f9fb1323e2c37f6ee4c7909e39ce016b2e8360d461705"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e0cb3102d21a2fe5f3210af608748ddd0cd09825ac12d42dc56ed5ed8725fe0"
+checksum = "7d361ed0373cf5f086b49c499aa72227b646a64f899f32e34312f97c0fadff75"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -120,15 +120,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72101dd1f441d629735143c41e00b3428f9267738176983ef588ff43382af0a0"
+checksum = "649782a39ce99798dd6b4029e2bb318a2fbeaade1b4fa25330763c10c65bc358"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6557f8ce44d498777f2495aa58d9692a4a37d6f84aa445750d666cef770b6a5c"
+checksum = "9c9909222db472fcc98d9e4e7192fa9d064dac63a3fa657df8c6daae86fb2604"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -145,19 +145,19 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88807e1c0c47ec02fe433333ccbe56b480425418b1470e333205e11650697d72"
+checksum = "68689b83e52e605ba48652882d3fccc2e2e136abf139eb64ae667888ba0d52f8"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
 ]
 
 [[package]]
 name = "cranelift-native"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c22b0d9fcbe3fc5a1af9e7021b44ce42b930bcefac446ce22e02e8f9a0d67120"
+checksum = "f98e4e99a353703475d5acb402b9c13482d41d8a4008b352559bd560afb90363"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -166,9 +166,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "341375758d7c3fedc0b5315f552e6f0feac46baf87c450a15e9455ef47c2b261"
+checksum = "b7a006ce1d8dd11df67567d8673e5920f3a56441812aed52a007ffce8f1b20e9"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -333,6 +333,7 @@ dependencies = [
  "cranelift-frontend",
  "cranelift-jit",
  "cranelift-module",
+ "cranelift-native",
  "cranelift-object",
  "gimli",
  "indexmap",
@@ -381,9 +382,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "5.0.0"
+version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08fcba5ebd96da2a9f0747ab6337fe9788adfb3f63fa2c180520d665562d257e"
+checksum = "ec1fd0f0dd79e7cc0f55b102e320d7c77ab76cd272008a8fd98e25b5777e2636"
 dependencies = [
  "cfg-if",
  "libc",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -15,14 +15,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.92", features = ["unwind", "all-arch"] }
-cranelift-frontend = { version = "0.92" }
-cranelift-module = { version = "0.92" }
-# NOTE vendored as src/cranelift_native.rs
-# FIXME revert back to the external crate with Cranelift 0.93
-#cranelift-native = { version = "0.92" }
-cranelift-jit = { version = "0.92", optional = true }
-cranelift-object = { version = "0.92" }
+cranelift-codegen = { version = "0.93", features = ["unwind", "all-arch"] }
+cranelift-frontend = { version = "0.93" }
+cranelift-module = { version = "0.93" }
+cranelift-native = { version = "0.93" }
+cranelift-jit = { version = "0.93", optional = true }
+cranelift-object = { version = "0.93" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.26.0", default-features = false, features = ["write"]}
 object = { version = "0.29.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock

@@ -50,9 +50,9 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.86"
+version = "0.1.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dae98c88e576098d7ab13ebcb40cc43e5114b2beafe61a87cda9200649ff205"
+checksum = "9fc9c2080d347a2c316518840ac9194644a9993dfa1e9778ef38979a339f5d8b"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -117,21 +117,20 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.2.6"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
 dependencies = [
  "compiler_builtins",
- "libc",
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
 ]
 
 [[package]]
 name = "libc"
-version = "0.2.139"
+version = "0.2.140"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -282,10 +281,8 @@ dependencies = [
 name = "test"
 version = "0.0.0"
 dependencies = [
- "cfg-if",
  "core",
  "getopts",
- "libc",
  "panic_abort",
  "panic_unwind",
  "proc_macro",

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml

@@ -8,7 +8,7 @@ alloc = { path = "./sysroot_src/library/alloc" }
 std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] }
 test = { path = "./sysroot_src/library/test" }
 
-compiler_builtins = { version = "0.1.39", default-features = false, features = ["no-asm"] }
+compiler_builtins = { version = "0.1.87", default-features = false, features = ["no-asm"] }
 
 [patch.crates-io]
 rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" }

compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs

@@ -46,13 +46,15 @@ pub(crate) fn build_sysroot(
         let wrapper_name = wrapper_base_name.replace("____", wrapper);
 
         let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
+        let wrapper_path = DIST_DIR.to_path(dirs).join(&wrapper_name);
         build_cargo_wrapper_cmd
             .env("TOOLCHAIN_NAME", toolchain_name.clone())
             .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
             .arg("-o")
-            .arg(DIST_DIR.to_path(dirs).join(wrapper_name))
+            .arg(&wrapper_path)
             .arg("-Cstrip=debuginfo");
         spawn_and_wait(build_cargo_wrapper_cmd);
+        try_hard_link(wrapper_path, BIN_DIR.to_path(dirs).join(wrapper_name));
     }
 
     let host = build_sysroot_for_triple(
@@ -247,6 +249,7 @@ fn build_clif_sysroot_for_triple(
     if channel == "release" {
         build_cmd.arg("--release");
     }
+    build_cmd.arg("--locked");
     build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
     if compiler.triple.contains("apple") {
         build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed");

compiler/rustc_codegen_cranelift/build_system/prepare.rs

@@ -6,15 +6,18 @@ use std::process::Command;
 use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
 use super::path::{Dirs, RelPath};
 use super::rustc_info::{get_default_sysroot, get_rustc_version};
+use super::tests::LIBCORE_TESTS_SRC;
 use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
 
 pub(crate) fn prepare(dirs: &Dirs) {
     RelPath::DOWNLOAD.ensure_fresh(dirs);
 
     spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs));
 
-    prepare_sysroot(dirs);
+    prepare_stdlib(dirs);
     spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs));
+
+    prepare_coretests(dirs);
     spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs));
 
     super::tests::RAND_REPO.fetch(dirs);
@@ -25,11 +28,11 @@ pub(crate) fn prepare(dirs: &Dirs) {
     spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs));
 }
 
-fn prepare_sysroot(dirs: &Dirs) {
+fn prepare_stdlib(dirs: &Dirs) {
     let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
     assert!(sysroot_src_orig.exists());
 
-    eprintln!("[COPY] sysroot src");
+    eprintln!("[COPY] stdlib src");
 
     // FIXME ensure builds error out or update the copy if any of the files copied here change
     BUILD_SYSROOT.ensure_fresh(dirs);
@@ -47,7 +50,25 @@ fn prepare_sysroot(dirs: &Dirs) {
     eprintln!("[GIT] init");
     init_git_repo(&SYSROOT_SRC.to_path(dirs));
 
-    apply_patches(dirs, "sysroot", &SYSROOT_SRC.to_path(dirs));
+    apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs));
+}
+
+fn prepare_coretests(dirs: &Dirs) {
+    let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
+    assert!(sysroot_src_orig.exists());
+
+    eprintln!("[COPY] coretests src");
+
+    fs::create_dir_all(LIBCORE_TESTS_SRC.to_path(dirs)).unwrap();
+    copy_dir_recursively(
+        &sysroot_src_orig.join("library/core/tests"),
+        &LIBCORE_TESTS_SRC.to_path(dirs),
+    );
+
+    eprintln!("[GIT] init");
+    init_git_repo(&LIBCORE_TESTS_SRC.to_path(dirs));
+
+    apply_patches(dirs, "coretests", &LIBCORE_TESTS_SRC.to_path(dirs));
 }
 
 pub(crate) struct GitRepo {

compiler/rustc_codegen_cranelift/build_system/tests.rs

@@ -1,4 +1,4 @@
-use super::build_sysroot::{self, SYSROOT_SRC};
+use super::build_sysroot;
 use super::config;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
@@ -94,40 +94,42 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]),
 ];
 
+// FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is
+// fixed.
 pub(crate) static RAND_REPO: GitRepo =
-    GitRepo::github("rust-random", "rand", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", "rand");
+    GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand");
 
 pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand");
 
 pub(crate) static REGEX_REPO: GitRepo =
-    GitRepo::github("rust-lang", "regex", "341f207c1071f7290e3f228c710817c280c8dca1", "regex");
+    GitRepo::github("rust-lang", "regex", "a9b2e02352db92ce1f6e5b7ecd41b8bbffbe161a", "regex");
 
 pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex");
 
 pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "portable-simd",
-    "582239ac3b32007613df04d7ffa78dc30f4c5645",
+    "9bd30e77b3a3c699af102ebb3df0f6110f8aa02e",
     "portable-simd",
 );
 
 pub(crate) static PORTABLE_SIMD: CargoProject =
     CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd");
 
-pub(crate) static LIBCORE_TESTS: CargoProject =
-    CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests");
+pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::DOWNLOAD.join("coretests_src");
+
+pub(crate) static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests");
 
 const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::custom("test.rust-random/rand", &|runner| {
         RAND.clean(&runner.dirs);
 
         if runner.is_native {
-            eprintln!("[TEST] rust-random/rand");
             let mut test_cmd = RAND.test(&runner.target_compiler, &runner.dirs);
-            test_cmd.arg("--workspace");
+            test_cmd.arg("--workspace").arg("--").arg("-q");
             spawn_and_wait(test_cmd);
         } else {
-            eprintln!("[AOT] rust-random/rand");
+            eprintln!("Cross-Compiling: Not running tests");
             let mut build_cmd = RAND.build(&runner.target_compiler, &runner.dirs);
             build_cmd.arg("--workspace").arg("--tests");
             spawn_and_wait(build_cmd);
@@ -137,7 +139,9 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         LIBCORE_TESTS.clean(&runner.dirs);
 
         if runner.is_native {
-            spawn_and_wait(LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs));
+            let mut test_cmd = LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs);
+            test_cmd.arg("--").arg("-q");
+            spawn_and_wait(test_cmd);
         } else {
             eprintln!("Cross-Compiling: Not running tests");
             let mut build_cmd = LIBCORE_TESTS.build(&runner.target_compiler, &runner.dirs);
@@ -148,18 +152,13 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::custom("test.regex-shootout-regex-dna", &|runner| {
         REGEX.clean(&runner.dirs);
 
-        // newer aho_corasick versions throw a deprecation warning
-        let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
-
         let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs);
         build_cmd.arg("--example").arg("shootout-regex-dna");
-        build_cmd.env("RUSTFLAGS", lint_rust_flags.clone());
         spawn_and_wait(build_cmd);
 
         if runner.is_native {
             let mut run_cmd = REGEX.run(&runner.target_compiler, &runner.dirs);
             run_cmd.arg("--example").arg("shootout-regex-dna");
-            run_cmd.env("RUSTFLAGS", lint_rust_flags);
 
             let input = fs::read_to_string(
                 REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-input.txt"),
@@ -171,13 +170,6 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             .unwrap();
 
             let output = spawn_and_wait_with_input(run_cmd, input);
-            // Make sure `[codegen mono items] start` doesn't poison the diff
-            let output = output
-                .lines()
-                .filter(|line| !line.contains("codegen mono items"))
-                .chain(Some("")) // This just adds the trailing newline
-                .collect::<Vec<&str>>()
-                .join("\r\n");
 
             let output_matches = expected.lines().eq(output.lines());
             if !output_matches {
@@ -192,27 +184,14 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::custom("test.regex", &|runner| {
         REGEX.clean(&runner.dirs);
 
-        // newer aho_corasick versions throw a deprecation warning
-        let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
-
         if runner.is_native {
             let mut run_cmd = REGEX.test(&runner.target_compiler, &runner.dirs);
-            run_cmd.args([
-                "--tests",
-                "--",
-                "--exclude-should-panic",
-                "--test-threads",
-                "1",
-                "-Zunstable-options",
-                "-q",
-            ]);
-            run_cmd.env("RUSTFLAGS", lint_rust_flags);
+            run_cmd.args(["--workspace", "--", "-q"]);
             spawn_and_wait(run_cmd);
         } else {
             eprintln!("Cross-Compiling: Not running tests");
             let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs);
             build_cmd.arg("--tests");
-            build_cmd.env("RUSTFLAGS", lint_rust_flags.clone());
             spawn_and_wait(build_cmd);
         }
     }),

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -518,6 +518,17 @@ pub struct Box<T: ?Sized>(Unique<T>, ());
 
 impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<Box<U>> for Box<T> {}
 
+impl<T> Box<T> {
+    pub fn new(val: T) -> Box<T> {
+        unsafe {
+            let size = intrinsics::size_of::<T>();
+            let ptr = libc::malloc(size);
+            intrinsics::copy(&val as *const T as *const u8, ptr, size);
+            Box(Unique { pointer: NonNull(ptr as *const T), _marker: PhantomData }, ())
+        }
+    }
+}
+
 impl<T: ?Sized> Drop for Box<T> {
     fn drop(&mut self) {
         // drop is currently performed by compiler.

compiler/rustc_codegen_cranelift/example/std_example.rs

@@ -58,8 +58,9 @@ fn main() {
     assert_eq!(0b0000000000000000000000000010000000000000000000000000000000000000_0000000000000000000000000000000000001000000000000000000010000000u128.trailing_zeros(), 7);
     assert_eq!(core::intrinsics::saturating_sub(0, -170141183460469231731687303715884105728i128), 170141183460469231731687303715884105727i128);
 
-    let _d = 0i128.checked_div(2i128);
-    let _d = 0u128.checked_div(2u128);
+    std::hint::black_box(std::hint::black_box(7571400400375753350092698930310845914i128) * 10);
+    assert!(0i128.checked_div(2i128).is_some());
+    assert!(0u128.checked_div(2u128).is_some());
     assert_eq!(1u128 + 2, 3);
 
     assert_eq!(0b100010000000000000000000000000000u128 >> 10, 0b10001000000000000000000u128);

compiler/rustc_codegen_cranelift/patches/0003-rand-Disable-rand-tests-on-mingw.patch

@@ -19,8 +19,8 @@ index 217899e..9cedeb7 100644
 +    // This is broken on x86_64-pc-windows-gnu presumably due to a broken powf implementation
 +    #[cfg_attr(all(target_os = "windows", target_env = "gnu"), ignore)]
      fn value_stability() {
-         fn test_samples<F: Float + core::fmt::Debug, D: Distribution<F>>(
-             distr: D, zero: F, expected: &[F],
+         fn test_samples<F: Float + Debug + Display + LowerExp, D: Distribution<F>>(
+             distr: D, thresh: F, expected: &[F],
 diff --git a/rand_distr/tests/value_stability.rs b/rand_distr/tests/value_stability.rs
 index 192ba74..0101ace 100644
 --- a/rand_distr/tests/value_stability.rs

compiler/rustc_codegen_cranelift/patches/0022-sysroot-Disable-not-compiling-tests.patch → compiler/rustc_codegen_cranelift/patches/0022-coretests-Disable-not-compiling-tests.patch

@@ -13,14 +13,14 @@ Subject: [PATCH] [core] Disable not compiling tests
  6 files changed, 16 insertions(+), 1 deletion(-)
  create mode 100644 library/core/tests/Cargo.toml
 
-diff --git a/library/core/tests/Cargo.toml b/library/core/tests/Cargo.toml
+diff --git a/Cargo.toml b/Cargo.toml
 new file mode 100644
 index 0000000..46fd999
 --- /dev/null
-+++ b/library/core/tests/Cargo.toml
++++ b/Cargo.toml
 @@ -0,0 +1,12 @@
 +[package]
-+name = "core"
++name = "coretests"
 +version = "0.0.0"
 +edition = "2021"
 +
@@ -31,5 +31,14 @@ index 0000000..46fd999
 +[dependencies]
 +rand = { version = "0.8.5", default-features = false }
 +rand_xorshift = { version = "0.3.0", default-features = false }
+diff --git a/lib.rs b/lib.rs
+index 42a26ae..5ac1042 100644
+--- a/lib.rs
++++ b/lib.rs
+@@ -1,3 +1,4 @@
++#![cfg(test)]
+ #![feature(alloc_layout_extra)]
+ #![feature(array_chunks)]
+ #![feature(array_methods)]
 --
 2.21.0 (Apple Git-122)

compiler/rustc_codegen_cranelift/patches/0023-sysroot-Ignore-failing-tests.patch → compiler/rustc_codegen_cranelift/patches/0023-coretests-Ignore-failing-tests.patch

@@ -10,10 +10,10 @@ Subject: [PATCH] [core] Ignore failing tests
  library/core/tests/time.rs       |  1 +
  4 files changed, 18 insertions(+), 2 deletions(-)
 
-diff --git a/library/core/tests/array.rs b/library/core/tests/array.rs
+diff --git a/array.rs b/array.rs
 index 4bc44e9..8e3c7a4 100644
---- a/library/core/tests/array.rs
-+++ b/library/core/tests/array.rs
+--- a/array.rs
++++ b/array.rs
 @@ -242,6 +242,7 @@ fn iterator_drops() {
      assert_eq!(i.get(), 5);
  }
@@ -46,10 +46,10 @@ index 4bc44e9..8e3c7a4 100644
 
  #[test]
  fn cell_allows_array_cycle() {
-diff --git a/library/core/tests/atomic.rs b/library/core/tests/atomic.rs
+diff --git a/atomic.rs b/atomic.rs
 index 13b12db..96fe4b9 100644
---- a/library/core/tests/atomic.rs
-+++ b/library/core/tests/atomic.rs
+--- a/atomic.rs
++++ b/atomic.rs
 @@ -185,6 +185,7 @@ fn ptr_bitops() {
  }
 

compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch

@@ -0,0 +1,30 @@
+From ad7ffe71baba46865f2e65266ab025920dfdc20b Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Thu, 18 Feb 2021 18:45:28 +0100
+Subject: [PATCH] Disable 128bit atomic operations
+
+Cranelift doesn't support them yet
+---
+ library/core/src/panic/unwind_safe.rs |  6 -----
+ library/core/src/sync/atomic.rs       | 38 ---------------------------
+ library/core/tests/atomic.rs          |  4 ---
+ 4 files changed, 4 insertions(+), 50 deletions(-)
+
+diff --git a/atomic.rs b/atomic.rs
+index b735957..ea728b6 100644
+--- a/atomic.rs
++++ b/atomic.rs
+@@ -185,10 +185,6 @@ fn atomic_alignment() {
+     assert_eq!(align_of::<AtomicU64>(), size_of::<AtomicU64>());
+     #[cfg(target_has_atomic = "64")]
+     assert_eq!(align_of::<AtomicI64>(), size_of::<AtomicI64>());
+-    #[cfg(target_has_atomic = "128")]
+-    assert_eq!(align_of::<AtomicU128>(), size_of::<AtomicU128>());
+-    #[cfg(target_has_atomic = "128")]
+-    assert_eq!(align_of::<AtomicI128>(), size_of::<AtomicI128>());
+     #[cfg(target_has_atomic = "ptr")]
+     assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>());
+     #[cfg(target_has_atomic = "ptr")]
+--
+2.26.2.7.g19db9cfb68
+

compiler/rustc_codegen_cranelift/patches/0027-sysroot-128bit-atomic-operations.patch → compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch

@@ -85,21 +85,6 @@ index d9de37e..8293fce 100644
 
  macro_rules! atomic_int_ptr_sized {
      ( $($target_pointer_width:literal $align:literal)* ) => { $(
-diff --git a/library/core/tests/atomic.rs b/library/core/tests/atomic.rs
-index b735957..ea728b6 100644
---- a/library/core/tests/atomic.rs
-+++ b/library/core/tests/atomic.rs
-@@ -185,10 +185,6 @@ fn atomic_alignment() {
-     assert_eq!(align_of::<AtomicU64>(), size_of::<AtomicU64>());
-     #[cfg(target_has_atomic = "64")]
-     assert_eq!(align_of::<AtomicI64>(), size_of::<AtomicI64>());
--    #[cfg(target_has_atomic = "128")]
--    assert_eq!(align_of::<AtomicU128>(), size_of::<AtomicU128>());
--    #[cfg(target_has_atomic = "128")]
--    assert_eq!(align_of::<AtomicI128>(), size_of::<AtomicI128>());
-     #[cfg(target_has_atomic = "ptr")]
-     assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>());
-     #[cfg(target_has_atomic = "ptr")]
 --
 2.26.2.7.g19db9cfb68
 

compiler/rustc_codegen_cranelift/patches/0028-sysroot-Disable-long-running-tests.patch → compiler/rustc_codegen_cranelift/patches/0028-coretests-Disable-long-running-tests.patch

@@ -7,10 +7,10 @@ Subject: [PATCH] Disable long running tests
  library/core/tests/slice.rs | 2 ++
  1 file changed, 2 insertions(+)
 
-diff --git a/library/core/tests/slice.rs b/library/core/tests/slice.rs
+diff --git a/slice.rs b/slice.rs
 index 8402833..84592e0 100644
---- a/library/core/tests/slice.rs
-+++ b/library/core/tests/slice.rs
+--- a/slice.rs
++++ b/slice.rs
 @@ -1809,6 +1809,7 @@ fn sort_unstable() {
      assert!(v == [0xDEADBEEF]);
  }

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-02-06"
+channel = "nightly-2023-03-15"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs

@@ -1,11 +1,14 @@
 use std::env;
 #[cfg(unix)]
 use std::os::unix::process::CommandExt;
-use std::path::PathBuf;
 use std::process::Command;
 
 fn main() {
-    let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+    let current_exe = env::current_exe().unwrap();
+    let mut sysroot = current_exe.parent().unwrap();
+    if sysroot.file_name().unwrap().to_str().unwrap() == "bin" {
+        sysroot = sysroot.parent().unwrap();
+    }
 
     let mut rustflags = String::new();
     rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend=");

compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs

@@ -2,11 +2,14 @@ use std::env;
 use std::ffi::OsString;
 #[cfg(unix)]
 use std::os::unix::process::CommandExt;
-use std::path::PathBuf;
 use std::process::Command;
 
 fn main() {
-    let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+    let current_exe = env::current_exe().unwrap();
+    let mut sysroot = current_exe.parent().unwrap();
+    if sysroot.file_name().unwrap().to_str().unwrap() == "bin" {
+        sysroot = sysroot.parent().unwrap();
+    }
 
     let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join(
         env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX,

compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs

@@ -2,11 +2,14 @@ use std::env;
 use std::ffi::OsString;
 #[cfg(unix)]
 use std::os::unix::process::CommandExt;
-use std::path::PathBuf;
 use std::process::Command;
 
 fn main() {
-    let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+    let current_exe = env::current_exe().unwrap();
+    let mut sysroot = current_exe.parent().unwrap();
+    if sysroot.file_name().unwrap().to_str().unwrap() == "bin" {
+        sysroot = sysroot.parent().unwrap();
+    }
 
     let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join(
         env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX,

compiler/rustc_codegen_cranelift/scripts/rustup.sh

@@ -2,10 +2,24 @@
 
 set -e
 
+TOOLCHAIN=${TOOLCHAIN:-$(date +%Y-%m-%d)}
+
+function check_git_fixed_subtree() {
+    if [[ ! -e ./git-fixed-subtree.sh ]]; then
+        echo "Missing git-fixed-subtree.sh. Please run the following commands to download it:"
+        echo "curl --proto '=https' --tlsv1.2 -sSf https://github.com/raw/bjorn3/git/tqc-subtree-portable/contrib/subtree/git-subtree.sh -o git-fixed-subtree.sh"
+        echo "chmod u+x git-fixed-subtree.sh"
+        exit 1
+    fi
+    if [[ ! -x ./git-fixed-subtree.sh ]]; then
+        echo "git-fixed-subtree.sh is not executable. Please run the following command to make it executable:"
+        echo "chmod u+x git-fixed-subtree.sh"
+        exit 1
+    fi
+}
+
 case $1 in
     "prepare")
-        TOOLCHAIN=$(date +%Y-%m-%d)
-
         echo "=> Installing new nightly"
         rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
         sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
@@ -27,28 +41,35 @@ case $1 in
         git commit -m "Rustup to $(rustc -V)"
         ;;
     "push")
+        check_git_fixed_subtree
+
         cg_clif=$(pwd)
         pushd ../rust
         git pull origin master
         branch=sync_cg_clif-$(date +%Y-%m-%d)
         git checkout -b "$branch"
-        git subtree pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master
+        "$cg_clif/git-fixed-subtree.sh" pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master
         git push -u my "$branch"
 
         # immediately merge the merge commit into cg_clif to prevent merge conflicts when syncing
         # from rust-lang/rust later
-        git subtree push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
+        "$cg_clif/git-fixed-subtree.sh" push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
         popd
         git merge sync_from_rust
 	;;
     "pull")
+        check_git_fixed_subtree
+
+        RUST_VERS=$(curl "https://static.rust-lang.org/dist/$TOOLCHAIN/channel-rust-nightly-git-commit-hash.txt")
+        echo "Pulling $RUST_VERS ($TOOLCHAIN)"
+
         cg_clif=$(pwd)
         pushd ../rust
-        git pull origin master
-        rust_vers="$(git rev-parse HEAD)"
-        git subtree push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
+        git fetch origin master
+        git checkout "$RUST_VERS"
+        "$cg_clif/git-fixed-subtree.sh" push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
         popd
-        git merge sync_from_rust -m "Sync from rust $rust_vers"
+        git merge sync_from_rust -m "Sync from rust $RUST_VERS"
         git branch -d sync_from_rust
         ;;
     *)

compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh

@@ -10,7 +10,7 @@ git fetch
 git checkout -- .
 git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
 
-git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-sysroot-*.patch
+git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-stdlib-*.patch
 
 git apply - <<EOF
 diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
@@ -36,7 +36,7 @@ changelog-seen = 2
 ninja = false
 
 [build]
-rustc = "$(pwd)/../dist/rustc-clif"
+rustc = "$(pwd)/../dist/bin/rustc-clif"
 cargo = "$(rustup which cargo)"
 full-bootstrap = true
 local-rebuild = true

compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh

@@ -21,6 +21,7 @@ done
 
 git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
 git checkout -- tests/ui/proc-macro/pretty-print-hack/
+rm tests/ui/parser/unclosed-delimiter-in-dep.rs # submodule contains //~ERROR
 
 # missing features
 # ================
@@ -80,6 +81,7 @@ rm tests/ui/layout/valid_range_oob.rs # different ICE message
 
 rm tests/ui/consts/issue-miri-1910.rs # different error message
 rm tests/ui/consts/offset_ub.rs # same
+rm tests/ui/consts/const-eval/ub-slice-get-unchecked.rs # same
 rm tests/ui/intrinsics/panic-uninitialized-zeroed.rs # same
 rm tests/ui/lint/lint-const-item-mutation.rs # same
 rm tests/ui/pattern/usefulness/doc-hidden-non-exhaustive.rs # same
@@ -110,12 +112,9 @@ rm tests/incremental/spike-neg2.rs # same
 
 rm tests/ui/simd/intrinsic/generic-reduction-pass.rs # simd_reduce_add_unordered doesn't accept an accumulator for integer vectors
 
-rm tests/ui/simd/intrinsic/generic-as.rs # crash when accessing vector type filed (#1318)
+rm tests/ui/simd/intrinsic/generic-as.rs # crash when accessing vector type field (#1318)
 rm tests/ui/simd/simd-bitmask.rs # crash
 
-rm tests/ui/dyn-star/dyn-star-to-dyn.rs
-rm tests/ui/dyn-star/dispatch-on-pin-mut.rs
-
 # bugs in the test suite
 # ======================
 rm tests/ui/backtrace.rs # TODO warning

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -4,6 +4,8 @@ mod comments;
 mod pass_mode;
 mod returning;
 
+use std::borrow::Cow;
+
 use cranelift_module::ModuleError;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::FnAbiOf;
@@ -25,7 +27,7 @@ fn clif_sig_from_fn_abi<'tcx>(
 ) -> Signature {
     let call_conv = conv_to_call_conv(tcx.sess, fn_abi.conv, default_call_conv);
 
-    let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
+    let inputs = fn_abi.args.iter().flat_map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter());
 
     let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx);
     // Sometimes the first param is an pointer to the place where the return value needs to be stored.
@@ -116,7 +118,52 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         params: Vec<AbiParam>,
         returns: Vec<AbiParam>,
         args: &[Value],
-    ) -> &[Value] {
+    ) -> Cow<'_, [Value]> {
+        if self.tcx.sess.target.is_like_windows {
+            let (mut params, mut args): (Vec<_>, Vec<_>) =
+                params
+                    .into_iter()
+                    .zip(args)
+                    .map(|(param, &arg)| {
+                        if param.value_type == types::I128 {
+                            let arg_ptr = Pointer::stack_slot(self.bcx.create_sized_stack_slot(
+                                StackSlotData { kind: StackSlotKind::ExplicitSlot, size: 16 },
+                            ));
+                            arg_ptr.store(self, arg, MemFlags::trusted());
+                            (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self))
+                        } else {
+                            (param, arg)
+                        }
+                    })
+                    .unzip();
+
+            let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128;
+
+            if indirect_ret_val {
+                params.insert(0, AbiParam::new(self.pointer_type));
+                let ret_ptr =
+                    Pointer::stack_slot(self.bcx.create_sized_stack_slot(StackSlotData {
+                        kind: StackSlotKind::ExplicitSlot,
+                        size: 16,
+                    }));
+                args.insert(0, ret_ptr.get_addr(self));
+                self.lib_call_unadjusted(name, params, vec![], &args);
+                return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]);
+            } else {
+                return self.lib_call_unadjusted(name, params, returns, &args);
+            }
+        }
+
+        self.lib_call_unadjusted(name, params, returns, args)
+    }
+
+    pub(crate) fn lib_call_unadjusted(
+        &mut self,
+        name: &str,
+        params: Vec<AbiParam>,
+        returns: Vec<AbiParam>,
+        args: &[Value],
+    ) -> Cow<'_, [Value]> {
         let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
         let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
         let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
@@ -125,41 +172,11 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         }
         let call_inst = self.bcx.ins().call(func_ref, args);
         if self.clif_comments.enabled() {
-            self.add_comment(call_inst, format!("easy_call {}", name));
+            self.add_comment(call_inst, format!("lib_call {}", name));
         }
         let results = self.bcx.inst_results(call_inst);
         assert!(results.len() <= 2, "{}", results.len());
-        results
-    }
-
-    pub(crate) fn easy_call(
-        &mut self,
-        name: &str,
-        args: &[CValue<'tcx>],
-        return_ty: Ty<'tcx>,
-    ) -> CValue<'tcx> {
-        let (input_tys, args): (Vec<_>, Vec<_>) = args
-            .iter()
-            .map(|arg| {
-                (AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self))
-            })
-            .unzip();
-        let return_layout = self.layout_of(return_ty);
-        let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
-            tup.iter().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
-        } else {
-            vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
-        };
-        let ret_vals = self.lib_call(name, input_tys, return_tys, &args);
-        match *ret_vals {
-            [] => CValue::by_ref(
-                Pointer::const_addr(self, i64::from(self.pointer_type.bytes())),
-                return_layout,
-            ),
-            [val] => CValue::by_val(val, return_layout),
-            [val, extra] => CValue::by_val_pair(val, extra, return_layout),
-            _ => unreachable!(),
-        }
+        Cow::Borrowed(results)
     }
 }
 
@@ -275,10 +292,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
     self::comments::add_locals_header_comment(fx);
 
     for (local, arg_kind, ty) in func_params {
-        let layout = fx.layout_of(ty);
-
-        let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa;
-
         // While this is normally an optimization to prevent an unnecessary copy when an argument is
         // not mutated by the current function, this is necessary to support unsized arguments.
         if let ArgKind::Normal(Some(val)) = arg_kind {
@@ -300,6 +313,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
             }
         }
 
+        let layout = fx.layout_of(ty);
+        let is_ssa = ssa_analyzed[local].is_ssa(fx, ty);
         let place = make_local_place(fx, local, layout, is_ssa);
         assert_eq!(fx.local_map.push(place), local);
 
@@ -323,7 +338,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
         let ty = fx.monomorphize(fx.mir.local_decls[local].ty);
         let layout = fx.layout_of(ty);
 
-        let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa;
+        let is_ssa = ssa_analyzed[local].is_ssa(fx, ty);
 
         let place = make_local_place(fx, local, layout, is_ssa);
         assert_eq!(fx.local_map.push(place), local);
@@ -515,10 +530,9 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                 args.into_iter()
                     .enumerate()
                     .skip(if first_arg_override.is_some() { 1 } else { 0 })
-                    .map(|(i, arg)| {
+                    .flat_map(|(i, arg)| {
                         adjust_arg_for_abi(fx, arg.value, &fn_abi.args[i], arg.is_owned).into_iter()
-                    })
-                    .flatten(),
+                    }),
             )
             .collect::<Vec<Value>>();
 

compiler/rustc_codegen_cranelift/src/abi/returning.rs

@@ -14,7 +14,8 @@ pub(super) fn codegen_return_param<'tcx>(
 ) -> CPlace<'tcx> {
     let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode {
         PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => {
-            let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
+            let is_ssa =
+                ssa_analyzed[RETURN_PLACE].is_ssa(fx, fx.fn_abi.as_ref().unwrap().ret.layout.ty);
             (
                 super::make_local_place(
                     fx,

compiler/rustc_codegen_cranelift/src/analyze.rs

@@ -4,34 +4,30 @@ use crate::prelude::*;
 
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir::StatementKind::*;
+use rustc_middle::ty::Ty;
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub(crate) enum SsaKind {
     NotSsa,
-    Ssa,
+    MaybeSsa,
+}
+
+impl SsaKind {
+    pub(crate) fn is_ssa<'tcx>(self, fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
+        self == SsaKind::MaybeSsa && (fx.clif_type(ty).is_some() || fx.clif_pair_type(ty).is_some())
+    }
 }
 
 pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
-    let mut flag_map = fx
-        .mir
-        .local_decls
-        .iter()
-        .map(|local_decl| {
-            let ty = fx.monomorphize(local_decl.ty);
-            if fx.clif_type(ty).is_some() || fx.clif_pair_type(ty).is_some() {
-                SsaKind::Ssa
-            } else {
-                SsaKind::NotSsa
-            }
-        })
-        .collect::<IndexVec<Local, SsaKind>>();
+    let mut flag_map =
+        fx.mir.local_decls.iter().map(|_| SsaKind::MaybeSsa).collect::<IndexVec<Local, SsaKind>>();
 
     for bb in fx.mir.basic_blocks.iter() {
         for stmt in bb.statements.iter() {
             match &stmt.kind {
                 Assign(place_and_rval) => match &place_and_rval.1 {
                     Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => {
-                        not_ssa(&mut flag_map, place.local)
+                        flag_map[place.local] = SsaKind::NotSsa;
                     }
                     _ => {}
                 },
@@ -42,7 +38,3 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
 
     flag_map
 }
-
-fn not_ssa(flag_map: &mut IndexVec<Local, SsaKind>, local: Local) {
-    flag_map[local] = SsaKind::NotSsa;
-}

compiler/rustc_codegen_cranelift/src/base.rs

@@ -192,7 +192,7 @@ pub(crate) fn compile_fn(
                         let pass_times = cranelift_codegen::timing::take_current();
                         // Replace newlines with | as measureme doesn't allow control characters like
                         // newlines inside strings.
-                        recorder.record_arg(format!("{}", pass_times).replace("\n", " | "));
+                        recorder.record_arg(format!("{}", pass_times).replace('\n', " | "));
                         recording_args = true;
                     },
                 )
@@ -365,11 +365,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 fx.bcx.set_cold_block(failure);
 
                 if *expected {
-                    fx.bcx.ins().brz(cond, failure, &[]);
+                    fx.bcx.ins().brif(cond, target, &[], failure, &[]);
                 } else {
-                    fx.bcx.ins().brnz(cond, failure, &[]);
+                    fx.bcx.ins().brif(cond, failure, &[], target, &[]);
                 };
-                fx.bcx.ins().jump(target, &[]);
 
                 fx.bcx.switch_to_block(failure);
                 fx.bcx.ins().nop();
@@ -425,11 +424,9 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                         }
                     } else {
                         if test_zero {
-                            fx.bcx.ins().brz(discr, then_block, &[]);
-                            fx.bcx.ins().jump(else_block, &[]);
+                            fx.bcx.ins().brif(discr, else_block, &[], then_block, &[]);
                         } else {
-                            fx.bcx.ins().brnz(discr, then_block, &[]);
-                            fx.bcx.ins().jump(else_block, &[]);
+                            fx.bcx.ins().brif(discr, then_block, &[], else_block, &[]);
                         }
                     }
                 } else {
@@ -750,8 +747,7 @@ fn codegen_stmt<'tcx>(
 
                         fx.bcx.switch_to_block(loop_block);
                         let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64);
-                        fx.bcx.ins().brnz(done, done_block, &[]);
-                        fx.bcx.ins().jump(loop_block2, &[]);
+                        fx.bcx.ins().brif(done, done_block, &[], loop_block2, &[]);
 
                         fx.bcx.switch_to_block(loop_block2);
                         let to = lval.place_index(fx, index);
@@ -997,7 +993,7 @@ fn codegen_panic_inner<'tcx>(
     let symbol_name = fx.tcx.symbol_name(instance).name;
 
     fx.lib_call(
-        &*symbol_name,
+        symbol_name,
         args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(),
         vec![],
         args,

compiler/rustc_codegen_cranelift/src/cast.rs

@@ -64,17 +64,12 @@ pub(crate) fn clif_int_or_float_cast(
                 },
             );
 
-            let from_rust_ty = if from_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-
-            let to_rust_ty = match to_ty {
-                types::F32 => fx.tcx.types.f32,
-                types::F64 => fx.tcx.types.f64,
-                _ => unreachable!(),
-            };
-
-            return fx
-                .easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
-                .load_scalar(fx);
+            return fx.lib_call(
+                &name,
+                vec![AbiParam::new(types::I128)],
+                vec![AbiParam::new(to_ty)],
+                &[from],
+            )[0];
         }
 
         // int-like -> float
@@ -101,16 +96,29 @@ pub(crate) fn clif_int_or_float_cast(
                 },
             );
 
-            let from_rust_ty = match from_ty {
-                types::F32 => fx.tcx.types.f32,
-                types::F64 => fx.tcx.types.f64,
-                _ => unreachable!(),
-            };
-
-            let to_rust_ty = if to_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-
-            fx.easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
-                .load_scalar(fx)
+            if fx.tcx.sess.target.is_like_windows {
+                let ret = fx.lib_call(
+                    &name,
+                    vec![AbiParam::new(from_ty)],
+                    vec![AbiParam::new(types::I64X2)],
+                    &[from],
+                )[0];
+                // FIXME use bitcast instead of store to get from i64x2 to i128
+                let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
+                    kind: StackSlotKind::ExplicitSlot,
+                    size: 16,
+                });
+                let ret_ptr = Pointer::stack_slot(stack_slot);
+                ret_ptr.store(fx, ret, MemFlags::trusted());
+                ret_ptr.load(fx, types::I128, MemFlags::trusted())
+            } else {
+                fx.lib_call(
+                    &name,
+                    vec![AbiParam::new(from_ty)],
+                    vec![AbiParam::new(types::I128)],
+                    &[from],
+                )[0]
+            }
         } else if to_ty == types::I8 || to_ty == types::I16 {
             // FIXME implement fcvt_to_*int_sat.i8/i16
             let val = if to_signed {

compiler/rustc_codegen_cranelift/src/codegen_i128.rs

@@ -29,39 +29,24 @@ pub(crate) fn maybe_codegen<'tcx>(
         BinOp::Add | BinOp::Sub if !checked => None,
         BinOp::Mul if !checked || is_signed => {
             if !checked {
-                let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-                if fx.tcx.sess.target.is_like_windows {
-                    let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
-                    let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
-                    let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
-                    assert!(lhs_extra.is_none());
-                    assert!(rhs_extra.is_none());
-                    let args = [
-                        ret_place.to_ptr().get_addr(fx),
-                        lhs_ptr.get_addr(fx),
-                        rhs_ptr.get_addr(fx),
-                    ];
-                    fx.lib_call(
-                        "__multi3",
-                        vec![
-                            AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn),
-                            AbiParam::new(fx.pointer_type),
-                            AbiParam::new(fx.pointer_type),
-                        ],
-                        vec![],
-                        &args,
-                    );
-                    Some(ret_place.to_cvalue(fx))
-                } else {
-                    Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
-                }
+                let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)];
+                let ret_val = fx.lib_call(
+                    "__multi3",
+                    vec![AbiParam::new(types::I128), AbiParam::new(types::I128)],
+                    vec![AbiParam::new(types::I128)],
+                    &args,
+                )[0];
+                Some(CValue::by_val(
+                    ret_val,
+                    fx.layout_of(if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 }),
+                ))
             } else {
                 let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]);
                 let oflow = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32));
                 let lhs = lhs.load_scalar(fx);
                 let rhs = rhs.load_scalar(fx);
                 let oflow_ptr = oflow.to_ptr().get_addr(fx);
-                let res = fx.lib_call(
+                let res = fx.lib_call_unadjusted(
                     "__muloti4",
                     vec![
                         AbiParam::new(types::I128),
@@ -80,29 +65,12 @@ pub(crate) fn maybe_codegen<'tcx>(
             assert!(checked);
             let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]);
             let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
-            let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
-                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
-                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
-                assert!(lhs_extra.is_none());
-                assert!(rhs_extra.is_none());
-                (
-                    vec![
-                        AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn),
-                        AbiParam::new(fx.pointer_type),
-                        AbiParam::new(fx.pointer_type),
-                    ],
-                    [out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)],
-                )
-            } else {
-                (
-                    vec![
-                        AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn),
-                        AbiParam::new(types::I128),
-                        AbiParam::new(types::I128),
-                    ],
-                    [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)],
-                )
-            };
+            let param_types = vec![
+                AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn),
+                AbiParam::new(types::I128),
+                AbiParam::new(types::I128),
+            ];
+            let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
             let name = match (bin_op, is_signed) {
                 (BinOp::Add, false) => "__rust_u128_addo",
                 (BinOp::Add, true) => "__rust_i128_addo",
@@ -125,14 +93,10 @@ pub(crate) fn maybe_codegen<'tcx>(
                 _ => unreachable!(),
             };
             if fx.tcx.sess.target.is_like_windows {
-                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
-                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
-                assert!(lhs_extra.is_none());
-                assert!(rhs_extra.is_none());
-                let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)];
                 let ret = fx.lib_call(
                     name,
-                    vec![AbiParam::new(fx.pointer_type), AbiParam::new(fx.pointer_type)],
+                    vec![AbiParam::new(types::I128), AbiParam::new(types::I128)],
                     vec![AbiParam::new(types::I64X2)],
                     &args,
                 )[0];
@@ -141,7 +105,14 @@ pub(crate) fn maybe_codegen<'tcx>(
                 ret_place.to_ptr().store(fx, ret, MemFlags::trusted());
                 Some(ret_place.to_cvalue(fx))
             } else {
-                Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty))
+                let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)];
+                let ret_val = fx.lib_call(
+                    name,
+                    vec![AbiParam::new(types::I128), AbiParam::new(types::I128)],
+                    vec![AbiParam::new(types::I128)],
+                    &args,
+                )[0];
+                Some(CValue::by_val(ret_val, lhs.layout()))
             }
         }
         BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => {

compiler/rustc_codegen_cranelift/src/common.rs

@@ -75,7 +75,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
         ty::Adt(adt_def, _) if adt_def.repr().simd() => {
             let (element, count) = match &tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().abi
             {
-                Abi::Vector { element, count } => (element.clone(), *count),
+                Abi::Vector { element, count } => (*element, *count),
                 _ => unreachable!(),
             };
 

compiler/rustc_codegen_cranelift/src/compiler_builtins.rs

@@ -39,6 +39,7 @@ builtin_functions! {
 
     // integers
     fn __multi3(a: i128, b: i128) -> i128;
+    fn __muloti4(n: i128, d: i128, oflow: &mut i32) -> i128;
     fn __udivti3(n: u128, d: u128) -> u128;
     fn __divti3(n: i128, d: i128) -> i128;
     fn __umodti3(n: u128, d: u128) -> u128;

compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs

@@ -32,7 +32,7 @@ impl ConcurrencyLimiter {
         ConcurrencyLimiter {
             helper_thread: Some(helper_thread),
             state,
-            available_token_condvar: Arc::new(Condvar::new()),
+            available_token_condvar,
             finished: false,
         }
     }

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -290,7 +290,7 @@ fn data_id_for_static(
         };
 
         let data_id = match module.declare_data(
-            &*symbol_name,
+            symbol_name,
             linkage,
             is_mutable,
             attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL),
@@ -338,7 +338,7 @@ fn data_id_for_static(
     };
 
     let data_id = match module.declare_data(
-        &*symbol_name,
+        symbol_name,
         linkage,
         is_mutable,
         attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL),

compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs

@@ -113,7 +113,7 @@ impl Writer for WriterRelocate {
                     offset: offset as u32,
                     size,
                     name: DebugRelocName::Symbol(symbol),
-                    addend: addend as i64,
+                    addend,
                     kind: object::RelocationKind::Absolute,
                 });
                 self.write_udata(0, size)

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -377,7 +377,7 @@ pub(crate) fn run_aot(
     };
 
     if tcx.dep_graph.is_fully_enabled() {
-        for cgu in &*cgus {
+        for cgu in cgus {
             tcx.ensure().codegen_unit(cgu.name());
         }
     }
@@ -417,7 +417,7 @@ pub(crate) fn run_aot(
                     CguReuse::PreLto => unreachable!(),
                     CguReuse::PostLto => {
                         concurrency_limiter.job_already_done();
-                        OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, &*cgu))
+                        OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu))
                     }
                 }
             })

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -311,7 +311,11 @@ fn dep_symbol_lookup_fn(
         .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable)
         .unwrap()
         .1;
-    for &cnum in &crate_info.used_crates {
+    // `used_crates` is in reverse postorder in terms of dependencies. Reverse the order here to
+    // get a postorder which ensures that all dependencies of a dylib are loaded before the dylib
+    // itself. This helps the dynamic linker to find dylibs not in the regular dynamic library
+    // search path.
+    for &cnum in crate_info.used_crates.iter().rev() {
         let src = &crate_info.used_crate_source[&cnum];
         match data[cnum.as_usize() - 1] {
             Linkage::NotLinked | Linkage::IncludedFromDylib => {}

compiler/rustc_codegen_cranelift/src/global_asm.rs

@@ -125,7 +125,7 @@ pub(crate) fn compile_global_asm(
     let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name));
 
     // Assemble `global_asm`
-    let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
+    let global_asm_object_file = add_file_stem_postfix(output_object_file, ".asm");
     let mut child = Command::new(&config.assembler)
         .arg("-o")
         .arg(&global_asm_object_file)

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -242,7 +242,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                 }
             }
             InlineAsmOperand::Const { ref value } => {
-                let (const_value, ty) = crate::constant::eval_mir_constant(fx, &*value)
+                let (const_value, ty) = crate::constant::eval_mir_constant(fx, value)
                     .unwrap_or_else(|| span_bug!(span, "asm const cannot be resolved"));
                 let value = rustc_codegen_ssa::common::asm_const_to_str(
                     fx.tcx,
@@ -334,13 +334,13 @@ pub(crate) fn codegen_inline_asm<'tcx>(
             }
             CInlineAsmOperand::Out { reg: _, late: _, place } => {
                 if let Some(place) = place {
-                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), place.clone()));
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
                 }
             }
             CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
                 inputs.push((asm_gen.stack_slots_input[i].unwrap(), in_value.load_scalar(fx)));
                 if let Some(out_place) = out_place {
-                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), out_place.clone()));
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), *out_place));
                 }
             }
             CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {}

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -23,7 +23,7 @@ pub(crate) use llvm::codegen_llvm_intrinsic_call;
 
 use rustc_middle::ty;
 use rustc_middle::ty::layout::{HasParamEnv, ValidityRequirement};
-use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_span::symbol::{kw, sym, Symbol};
 
@@ -252,45 +252,45 @@ fn codegen_float_intrinsic_call<'tcx>(
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
 ) -> bool {
-    let (name, arg_count, ty) = match intrinsic {
-        sym::expf32 => ("expf", 1, fx.tcx.types.f32),
-        sym::expf64 => ("exp", 1, fx.tcx.types.f64),
-        sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
-        sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
-        sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
-        sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
-        sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
-        sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
-        sym::powf32 => ("powf", 2, fx.tcx.types.f32),
-        sym::powf64 => ("pow", 2, fx.tcx.types.f64),
-        sym::logf32 => ("logf", 1, fx.tcx.types.f32),
-        sym::logf64 => ("log", 1, fx.tcx.types.f64),
-        sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
-        sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
-        sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
-        sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
-        sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
-        sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
-        sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
-        sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
-        sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
-        sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
-        sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
-        sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
-        sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
-        sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
-        sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
-        sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
-        sym::rintf32 => ("rintf", 1, fx.tcx.types.f32),
-        sym::rintf64 => ("rint", 1, fx.tcx.types.f64),
-        sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
-        sym::roundf64 => ("round", 1, fx.tcx.types.f64),
-        sym::roundevenf32 => ("roundevenf", 1, fx.tcx.types.f32),
-        sym::roundevenf64 => ("roundeven", 1, fx.tcx.types.f64),
-        sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
-        sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
-        sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
-        sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
+    let (name, arg_count, ty, clif_ty) = match intrinsic {
+        sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32),
+        sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64),
+        sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32),
+        sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64),
+        sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32),
+        sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64),
+        sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins
+        sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins
+        sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32),
+        sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64),
+        sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32),
+        sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64),
+        sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32),
+        sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64),
+        sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32),
+        sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64),
+        sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32),
+        sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64),
+        sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
+        sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
+        sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32),
+        sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64),
+        sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32),
+        sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64),
+        sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32),
+        sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64),
+        sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32),
+        sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64),
+        sym::rintf32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
+        sym::rintf64 => ("rint", 1, fx.tcx.types.f64, types::F64),
+        sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32),
+        sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
+        sym::roundevenf32 => ("roundevenf", 1, fx.tcx.types.f32, types::F32),
+        sym::roundevenf64 => ("roundeven", 1, fx.tcx.types.f64, types::F64),
+        sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32),
+        sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64),
+        sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32),
+        sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64),
         _ => return false,
     };
 
@@ -301,15 +301,19 @@ fn codegen_float_intrinsic_call<'tcx>(
     let (a, b, c);
     let args = match args {
         [x] => {
-            a = [codegen_operand(fx, x)];
+            a = [codegen_operand(fx, x).load_scalar(fx)];
             &a as &[_]
         }
         [x, y] => {
-            b = [codegen_operand(fx, x), codegen_operand(fx, y)];
+            b = [codegen_operand(fx, x).load_scalar(fx), codegen_operand(fx, y).load_scalar(fx)];
             &b
         }
         [x, y, z] => {
-            c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
+            c = [
+                codegen_operand(fx, x).load_scalar(fx),
+                codegen_operand(fx, y).load_scalar(fx),
+                codegen_operand(fx, z).load_scalar(fx),
+            ];
             &c
         }
         _ => unreachable!(),
@@ -318,15 +322,10 @@ fn codegen_float_intrinsic_call<'tcx>(
     let layout = fx.layout_of(ty);
     let res = match intrinsic {
         sym::fmaf32 | sym::fmaf64 => {
-            let a = args[0].load_scalar(fx);
-            let b = args[1].load_scalar(fx);
-            let c = args[2].load_scalar(fx);
-            CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
+            CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout)
         }
         sym::copysignf32 | sym::copysignf64 => {
-            let a = args[0].load_scalar(fx);
-            let b = args[1].load_scalar(fx);
-            CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
+            CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout)
         }
         sym::fabsf32
         | sym::fabsf64
@@ -336,21 +335,29 @@ fn codegen_float_intrinsic_call<'tcx>(
         | sym::ceilf64
         | sym::truncf32
         | sym::truncf64 => {
-            let a = args[0].load_scalar(fx);
-
             let val = match intrinsic {
-                sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
-                sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
-                sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
-                sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
+                sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(args[0]),
+                sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
+                sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
+                sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
                 _ => unreachable!(),
             };
 
             CValue::by_val(val, layout)
         }
+
         // These intrinsics aren't supported natively by Cranelift.
         // Lower them to a libcall.
-        _ => fx.easy_call(name, &args, ty),
+        sym::powif32 | sym::powif64 => {
+            let input_tys: Vec<_> = vec![AbiParam::new(clif_ty), AbiParam::new(types::I32)];
+            let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
+            CValue::by_val(ret_val, fx.layout_of(ty))
+        }
+        _ => {
+            let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect();
+            let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
+            CValue::by_val(ret_val, fx.layout_of(ty))
+        }
     };
 
     ret.write_cvalue(fx, res);
@@ -385,7 +392,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
             fx.bcx.ins().debugtrap();
         }
-        sym::copy | sym::copy_nonoverlapping => {
+        sym::copy => {
             intrinsic_args!(fx, args => (src, dst, count); intrinsic);
             let src = src.load_scalar(fx);
             let dst = dst.load_scalar(fx);
@@ -397,13 +404,8 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let byte_amount =
                 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
 
-            if intrinsic == sym::copy_nonoverlapping {
-                // FIXME emit_small_memcpy
-                fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
-            } else {
-                // FIXME emit_small_memmove
-                fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
-            }
+            // FIXME emit_small_memmove
+            fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
         }
         sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => {
             // NOTE: the volatile variants have src and dst swapped
@@ -643,26 +645,25 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
                 if do_panic {
                     let layout = fx.layout_of(ty);
-
-                    with_no_trimmed_paths!({
-                        crate::base::codegen_panic_nounwind(
-                            fx,
-                            &if layout.abi.is_uninhabited() {
-                                format!("attempted to instantiate uninhabited type `{}`", layout.ty)
-                            } else if requirement == ValidityRequirement::Zero {
+                    let msg_str = with_no_visible_paths!({
+                        with_no_trimmed_paths!({
+                            if layout.abi.is_uninhabited() {
+                                // Use this error even for the other intrinsics as it is more precise.
+                                format!("attempted to instantiate uninhabited type `{}`", ty)
+                            } else if intrinsic == sym::assert_zero_valid {
                                 format!(
                                     "attempted to zero-initialize type `{}`, which is invalid",
-                                    layout.ty
+                                    ty
                                 )
                             } else {
                                 format!(
                                     "attempted to leave type `{}` uninitialized, which is invalid",
-                                    layout.ty
+                                    ty
                                 )
-                            },
-                            source_info,
-                        )
+                            }
+                        })
                     });
+                    crate::base::codegen_panic_nounwind(fx, &msg_str, source_info);
                     return;
                 }
             }

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -279,9 +279,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
                 let trap_block = fx.bcx.create_block();
                 let true_ = fx.bcx.ins().iconst(types::I8, 1);
-                fx.bcx.ins().brnz(true_, trap_block, &[]);
                 let ret_block = fx.get_block(target);
-                fx.bcx.ins().jump(ret_block, &[]);
+                fx.bcx.ins().brif(true_, trap_block, &[], ret_block, &[]);
                 fx.bcx.switch_to_block(trap_block);
                 crate::trap::trap_unimplemented(
                     fx,
@@ -825,8 +824,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let next = fx.bcx.create_block();
                 let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
 
-                fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
-                fx.bcx.ins().jump(if_disabled, &[]);
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]);
                 fx.bcx.seal_block(if_enabled);
                 fx.bcx.seal_block(if_disabled);
 
@@ -864,8 +862,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let if_enabled = fx.bcx.create_block();
                 let next = fx.bcx.create_block();
 
-                fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
-                fx.bcx.ins().jump(next, &[]);
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], next, &[]);
                 fx.bcx.seal_block(if_enabled);
 
                 fx.bcx.switch_to_block(if_enabled);

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -57,8 +57,6 @@ mod compiler_builtins;
 mod concurrency_limiter;
 mod config;
 mod constant;
-// FIXME revert back to the external crate with Cranelift 0.93
-mod cranelift_native;
 mod debuginfo;
 mod discriminant;
 mod driver;
@@ -251,7 +249,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple {
     }
 }
 
-fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> {
+fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::TargetIsa + 'static> {
     use target_lexicon::BinaryFormat;
 
     let target_triple = crate::target_triple(sess);
@@ -285,14 +283,17 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
         }
     }
 
-    if let target_lexicon::Architecture::Aarch64(_) | target_lexicon::Architecture::X86_64 =
-        target_triple.architecture
+    if let target_lexicon::Architecture::Aarch64(_)
+    | target_lexicon::Architecture::Riscv64(_)
+    | target_lexicon::Architecture::X86_64 = target_triple.architecture
     {
-        // Windows depends on stack probes to grow the committed part of the stack
+        // Windows depends on stack probes to grow the committed part of the stack.
+        // On other platforms it helps prevents stack smashing.
         flags_builder.enable("enable_probestack").unwrap();
         flags_builder.set("probestack_strategy", "inline").unwrap();
     } else {
-        // __cranelift_probestack is not provided and inline stack probes are only supported on AArch64 and x86_64
+        // __cranelift_probestack is not provided and inline stack probes are only supported on
+        // AArch64, Riscv64 and x86_64.
         flags_builder.set("enable_probestack", "false").unwrap();
     }
 

compiler/rustc_codegen_cranelift/src/main_shim.rs

@@ -28,7 +28,7 @@ pub(crate) fn maybe_create_entry_wrapper(
 
     if main_def_id.is_local() {
         let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
-        if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() {
+        if !is_jit && module.get_name(tcx.symbol_name(instance).name).is_none() {
             return;
         }
     } else if !is_primary_cgu {

compiler/rustc_codegen_cranelift/src/num.rs

@@ -347,12 +347,20 @@ pub(crate) fn codegen_float_binop<'tcx>(
         BinOp::Mul => b.fmul(lhs, rhs),
         BinOp::Div => b.fdiv(lhs, rhs),
         BinOp::Rem => {
-            let name = match in_lhs.layout().ty.kind() {
-                ty::Float(FloatTy::F32) => "fmodf",
-                ty::Float(FloatTy::F64) => "fmod",
+            let (name, ty) = match in_lhs.layout().ty.kind() {
+                ty::Float(FloatTy::F32) => ("fmodf", types::F32),
+                ty::Float(FloatTy::F64) => ("fmod", types::F64),
                 _ => bug!(),
             };
-            return fx.easy_call(name, &[in_lhs, in_rhs], in_lhs.layout().ty);
+
+            let ret_val = fx.lib_call(
+                name,
+                vec![AbiParam::new(ty), AbiParam::new(ty)],
+                vec![AbiParam::new(ty)],
+                &[lhs, rhs],
+            )[0];
+
+            return CValue::by_val(ret_val, in_lhs.layout());
         }
         BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
             let fltcc = match bin_op {

compiler/rustc_codegen_cranelift/src/pointer.rs

@@ -30,11 +30,6 @@ impl Pointer {
         Pointer { base: PointerBase::Stack(stack_slot), offset: Offset32::new(0) }
     }
 
-    pub(crate) fn const_addr(fx: &mut FunctionCx<'_, '_, '_>, addr: i64) -> Self {
-        let addr = fx.bcx.ins().iconst(fx.pointer_type, addr);
-        Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
-    }
-
     pub(crate) fn dangling(align: Align) -> Self {
         Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
     }

compiler/rustc_codegen_cranelift/src/pretty_clif.rs

@@ -245,7 +245,7 @@ pub(crate) fn write_clif_file(
         for flag in isa.flags().iter() {
             writeln!(file, "set {}", flag)?;
         }
-        write!(file, "target {}", isa.triple().architecture.to_string())?;
+        write!(file, "target {}", isa.triple().architecture)?;
         for isa_flag in isa.isa_flags().iter() {
             write!(file, " {}", isa_flag)?;
         }

compiler/rustc_codegen_cranelift/src/unsize.rs

@@ -28,9 +28,7 @@ pub(crate) fn unsized_info<'tcx>(
         (
             &ty::Dynamic(ref data_a, _, src_dyn_kind),
             &ty::Dynamic(ref data_b, _, target_dyn_kind),
-        ) => {
-            assert_eq!(src_dyn_kind, target_dyn_kind);
-
+        ) if src_dyn_kind == target_dyn_kind => {
             let old_info =
                 old_info.expect("unsized_info: missing old info for trait upcasting coercion");
             if data_a.principal_def_id() == data_b.principal_def_id() {
@@ -55,7 +53,7 @@ pub(crate) fn unsized_info<'tcx>(
                 old_info
             }
         }
-        (_, &ty::Dynamic(ref data, ..)) => crate::vtable::get_vtable(fx, source, data.principal()),
+        (_, ty::Dynamic(data, ..)) => crate::vtable::get_vtable(fx, source, data.principal()),
         _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),
     }
 }

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -3,6 +3,7 @@
 use crate::prelude::*;
 
 use cranelift_codegen::ir::immediates::Offset32;
+use cranelift_codegen::ir::{InstructionData, Opcode};
 
 fn codegen_field<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
@@ -457,13 +458,15 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 
+    #[track_caller]
     pub(crate) fn to_ptr(self) -> Pointer {
         match self.to_ptr_maybe_unsized() {
             (ptr, None) => ptr,
             (_, Some(_)) => bug!("Expected sized cplace, found {:?}", self),
         }
     }
 
+    #[track_caller]
     pub(crate) fn to_ptr_maybe_unsized(self) -> (Pointer, Option<Value>) {
         match self.inner {
             CPlaceInner::Addr(ptr, extra) => (ptr, extra),
@@ -787,7 +790,36 @@ impl<'tcx> CPlace<'tcx> {
         index: Value,
     ) -> CPlace<'tcx> {
         let (elem_layout, ptr) = match self.layout().ty.kind() {
-            ty::Array(elem_ty, _) => (fx.layout_of(*elem_ty), self.to_ptr()),
+            ty::Array(elem_ty, _) => {
+                let elem_layout = fx.layout_of(*elem_ty);
+                match self.inner {
+                    CPlaceInner::Var(local, var) => {
+                        // This is a hack to handle `vector_val.0[1]`. It doesn't allow dynamic
+                        // indexing.
+                        let lane_idx = match fx.bcx.func.dfg.insts
+                            [fx.bcx.func.dfg.value_def(index).unwrap_inst()]
+                        {
+                            InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => imm,
+                            _ => bug!(
+                                "Dynamic indexing into a vector type is not supported: {self:?}[{index}]"
+                            ),
+                        };
+                        return CPlace {
+                            inner: CPlaceInner::VarLane(
+                                local,
+                                var,
+                                lane_idx.bits().try_into().unwrap(),
+                            ),
+                            layout: elem_layout,
+                        };
+                    }
+                    CPlaceInner::Addr(addr, None) => (elem_layout, addr),
+                    CPlaceInner::Addr(_, Some(_))
+                    | CPlaceInner::VarPair(_, _, _)
+                    | CPlaceInner::VarLane(_, _, _) => bug!("Can't index into {self:?}"),
+                }
+                // FIXME use VarLane in case of Var with simd type
+            }
             ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_maybe_unsized().0),
             _ => bug!("place_index({:?})", self.layout().ty),
         };

compiler/rustc_codegen_cranelift/src/vtable.rs

@@ -43,10 +43,29 @@ pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -
 
 pub(crate) fn get_ptr_and_method_ref<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    arg: CValue<'tcx>,
+    mut arg: CValue<'tcx>,
     idx: usize,
 ) -> (Pointer, Value) {
     let (ptr, vtable) = 'block: {
+        if let Abi::Scalar(_) = arg.layout().abi {
+            'descend_newtypes: while !arg.layout().ty.is_unsafe_ptr()
+                && !arg.layout().ty.is_region_ptr()
+            {
+                for i in 0..arg.layout().fields.count() {
+                    let field = arg.value_field(fx, mir::Field::new(i));
+                    if !field.layout().is_zst() {
+                        // we found the one non-zero-sized field that is allowed
+                        // now find *its* non-zero-sized field, or stop if it's a
+                        // pointer
+                        arg = field;
+                        continue 'descend_newtypes;
+                    }
+                }
+
+                bug!("receiver has no non-zero-sized fields {:?}", arg);
+            }
+        }
+
         if let ty::Ref(_, ty, _) = arg.layout().ty.kind() {
             if ty.is_dyn_star() {
                 let inner_layout = fx.layout_of(arg.layout().ty.builtin_deref(true).unwrap().ty);