diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
index f524b42c5eecd..7b73d3c00e60e 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
@@ -65,6 +65,12 @@ jobs:
         git config --global user.name "User"
         ./y.rs prepare
 
+    - name: Build without unstable features
+      env:
+        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
+      # This is the config rust-lang/rust uses for builds
+      run: ./y.rs build --no-unstable-features
+
     - name: Build
       run: ./y.rs build --sysroot none
 
@@ -152,11 +158,12 @@ jobs:
 
         ./y.exe build
 
-    #- name: Package prebuilt cg_clif
-    #  run: tar cvfJ cg_clif.tar.xz build
+    - name: Package prebuilt cg_clif
+      # don't use compression as xzip isn't supported by tar on windows and bzip2 hangs
+      run: tar cvf cg_clif.tar build
 
-    #- name: Upload prebuilt cg_clif
-    #  uses: actions/upload-artifact@v2
-    #  with:
-    #    name: cg_clif-${{ runner.os }}
-    #    path: cg_clif.tar.xz
+    - name: Upload prebuilt cg_clif
+      uses: actions/upload-artifact@v2
+      with:
+        name: cg_clif-${{ runner.os }}
+        path: cg_clif.tar
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/nightly-cranelift.yml b/compiler/rustc_codegen_cranelift/.github/workflows/nightly-cranelift.yml
new file mode 100644
index 0000000000000..c5b96a4782804
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/nightly-cranelift.yml
@@ -0,0 +1,59 @@
+name: Test nightly Cranelift
+
+on:
+  push:
+  schedule:
+    - cron: '1 17 * * *' # At 01:17 UTC every day.
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ubuntu-latest-cargo-installed-crates
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./y.rs prepare
+
+    - name: Patch Cranelift
+      run: |
+        sed -i 's/cranelift-codegen = { version = "\w*.\w*.\w*", features = \["unwind", "all-arch"\] }/cranelift-codegen = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git", features = ["unwind", "all-arch"] }/' Cargo.toml
+        sed -i 's/cranelift-frontend = "\w*.\w*.\w*"/cranelift-frontend = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git" }/' Cargo.toml
+        sed -i 's/cranelift-module = "\w*.\w*.\w*"/cranelift-module = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git" }/' Cargo.toml
+        sed -i 's/cranelift-native = "\w*.\w*.\w*"/cranelift-native = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git" }/' Cargo.toml
+        sed -i 's/cranelift-jit = { version = "\w*.\w*.\w*", optional = true }/cranelift-jit = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git", optional = true }/' Cargo.toml
+        sed -i 's/cranelift-object = "\w*.\w*.\w*"/cranelift-object = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git" }/' Cargo.toml
+
+        sed -i 's/gimli = { version = "0.25.0", default-features = false, features = \["write"\]}/gimli = { version = "0.26.1", default-features = false, features = ["write"] }/' Cargo.toml
+
+        cat Cargo.toml
+
+    - name: Build without unstable features
+      # This is the config rust-lang/rust uses for builds
+      run: ./y.rs build --no-unstable-features
+
+    - name: Build
+      run: ./y.rs build --sysroot none
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        # Reduce amount of benchmark runs as they are slow
+        export COMPILE_RUNS=2
+        export RUN_RUNS=2
+
+        # Enable extra checks
+        export CG_CLIF_ENABLE_VERIFIER=1
+
+        ./test.sh
diff --git a/compiler/rustc_codegen_cranelift/.vscode/settings.json b/compiler/rustc_codegen_cranelift/.vscode/settings.json
index f62e59cefc241..74fde9c27c0e4 100644
--- a/compiler/rustc_codegen_cranelift/.vscode/settings.json
+++ b/compiler/rustc_codegen_cranelift/.vscode/settings.json
@@ -5,6 +5,7 @@
     "rust-analyzer.assist.importEnforceGranularity": true,
     "rust-analyzer.assist.importPrefix": "crate",
     "rust-analyzer.cargo.runBuildScripts": true,
+    "rust-analyzer.cargo.features": ["unstable-features"]
     "rust-analyzer.linkedProjects": [
         "./Cargo.toml",
         //"./build_sysroot/sysroot_src/src/libstd/Cargo.toml",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index 4afddf76869de..65e142a00f88e 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "anyhow"
-version = "1.0.42"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486"
+checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203"
 
 [[package]]
 name = "ar"
@@ -21,9 +21,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "bitflags"
-version = "1.2.1"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "cfg-if"
@@ -33,16 +33,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc0cb7df82c8cf8f2e6a8dd394a0932a71369c160cc9b027dca414fced242513"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe4463c15fa42eee909e61e5eac4866b7c6d22d0d8c621e57a0c5380753bfa8c"
 dependencies = [
  "cranelift-bforest",
  "cranelift-codegen-meta",
@@ -57,8 +59,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793f6a94a053a55404ea16e1700202a88101672b8cd6b4df63e13cde950852bf"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -66,18 +69,21 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44aa1846df275bce5eb30379d65964c7afc63c05a117076e62a119c25fe174be"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3a45d8d6318bf8fc518154d9298eab2a8154ec068a8885ff113f6db8d69bb3a"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e07339bd461766deb7605169de039e01954768ff730fa1254e149001884a8525"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -87,8 +93,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e8f0d60fb5d67f7a1e5c49db38ba96d1c846921faef02085fc5590b74781747"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -104,8 +111,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "825ac7e0959cbe7ddc9cc21209f0319e611a57f9fcb2b723861fe7ef2017e651"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -115,8 +123,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03e2fca76ff57e0532936a71e3fc267eae6a19a86656716479c66e7f912e3d7b"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -125,8 +134,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.76.0"
-source = "git+https://github.com/bytecodealliance/wasmtime.git#9c550fcf41425942ed97c747f0169b2ca81f9c1b"
+version = "0.78.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55500d0fc9bb05c0944fc4506649249d28f55bd4fe95b87f0e55bf41058f0e6d"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -138,9 +148,9 @@ dependencies = [
 
 [[package]]
 name = "crc32fast"
-version = "1.2.1"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
+checksum = "738c290dfaea84fc1ca15ad9c168d083b05a714e1efddd8edaab678dc28d2836"
 dependencies = [
  "cfg-if",
 ]
@@ -172,9 +182,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.98"
+version = "0.2.112"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
+checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
 
 [[package]]
 name = "libloading"
@@ -206,15 +216,15 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
 
 [[package]]
 name = "object"
-version = "0.26.0"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c55827317fb4c08822499848a14237d2874d6f139828893017237e7ab93eb386"
+checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
 dependencies = [
  "crc32fast",
  "indexmap",
@@ -223,9 +233,9 @@ dependencies = [
 
 [[package]]
 name = "regalloc"
-version = "0.0.31"
+version = "0.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "571f7f397d61c4755285cd37853fe8e03271c243424a907415909379659381c5"
+checksum = "a6304468554ed921da3d32c355ea107b8d13d7b8996c3adfb7aab48d3bc321f4"
 dependencies = [
  "log",
  "rustc-hash",
@@ -271,15 +281,15 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.6.1"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
+checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.1"
+version = "0.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0652da4c4121005e9ed22b79f6c5f2d9e2752906b53a33e9490489ba421a6fb"
+checksum = "d9bffcddbc2458fa3e6058414599e3c838a022abae82e5c67b4f7f80298d5bff"
 
 [[package]]
 name = "winapi"
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index 61d40702a3284..900411286b52e 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -8,23 +8,23 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", features = ["unwind", "all-arch"] }
-cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git" }
-cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git" }
-cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git" }
-cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", optional = true }
-cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git" }
+cranelift-codegen = { version = "0.78.0", features = ["unwind", "all-arch"] }
+cranelift-frontend = "0.78.0"
+cranelift-module = "0.78.0"
+cranelift-native = "0.78.0"
+cranelift-jit = { version = "0.78.0", optional = true }
+cranelift-object = "0.78.0"
 target-lexicon = "0.12.0"
 gimli = { version = "0.25.0", default-features = false, features = ["write"]}
-object = { version = "0.26.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
+object = { version = "0.27.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
 libloading = { version = "0.6.0", optional = true }
 smallvec = "1.6.1"
 
+[patch.crates-io]
 # Uncomment to use local checkout of cranelift
-#[patch."https://github.com/bytecodealliance/wasmtime.git"]
 #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }
 #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" }
 #cranelift-module = { path = "../wasmtime/cranelift/module" }
@@ -32,7 +32,6 @@ smallvec = "1.6.1"
 #cranelift-jit = { path = "../wasmtime/cranelift/jit" }
 #cranelift-object = { path = "../wasmtime/cranelift/object" }
 
-#[patch.crates-io]
 #gimli = { path = "../" }
 
 [features]
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
index 22be21cb8dee9..dd09656248022 100644
--- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
@@ -40,9 +40,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "cc"
-version = "1.0.70"
+version = "1.0.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0"
+checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
 
 [[package]]
 name = "cfg-if"
@@ -56,7 +56,7 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.50"
+version = "0.1.66"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -67,9 +67,9 @@ version = "0.0.0"
 
 [[package]]
 name = "dlmalloc"
-version = "0.2.1"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "332570860c2edf2d57914987bf9e24835425f75825086b6ba7d1e6a3e4f1f254"
+checksum = "a6fe28e0bf9357092740362502f5cc7955d8dc125ebda71dec72336c2e15c62e"
 dependencies = [
  "compiler_builtins",
  "libc",
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.102"
+version = "0.2.112"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103"
+checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
 dependencies = [
  "rustc-std-workspace-core",
 ]
diff --git a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
index 150b6d01a6b30..ccc50ee4a59bf 100644
--- a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
@@ -2,9 +2,17 @@ use std::env;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-pub(crate) fn build_backend(channel: &str, host_triple: &str) -> PathBuf {
+pub(crate) fn build_backend(
+    channel: &str,
+    host_triple: &str,
+    use_unstable_features: bool,
+) -> PathBuf {
     let mut cmd = Command::new("cargo");
-    cmd.arg("build").arg("--target").arg(host_triple).arg("--features").arg("unstable-features");
+    cmd.arg("build").arg("--target").arg(host_triple);
+
+    if use_unstable_features {
+        cmd.arg("--features").arg("unstable-features");
+    }
 
     match channel {
         "debug" => {}
diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
index 642abc41f45a7..1c78e7b5171ee 100644
--- a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
@@ -193,8 +193,7 @@ fn build_clif_sysroot_for_triple(
         "RUSTC",
         env::current_dir().unwrap().join(target_dir).join("bin").join("cg_clif_build_sysroot"),
     );
-    // FIXME Enable incremental again once rust-lang/rust#74946 is fixed
-    build_cmd.env("CARGO_INCREMENTAL", "0").env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
+    build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
     spawn_and_wait(build_cmd);
 
     // Copy all relevant files to the sysroot
diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs
index ae9a35048bda9..561e2ed7b0017 100644
--- a/compiler/rustc_codegen_cranelift/build_system/prepare.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs
@@ -30,7 +30,7 @@ pub(crate) fn prepare() {
     clone_repo(
         "portable-simd",
         "https://github.com/rust-lang/portable-simd",
-        "8cf7a62e5d2552961df51e5200aaa5b7c890a4bf",
+        "b8d6b6844602f80af79cd96401339ec594d472d8",
     );
     apply_patches("portable-simd", Path::new("portable-simd"));
 
@@ -92,7 +92,7 @@ fn prepare_sysroot() {
     clone_repo(
         "build_sysroot/compiler-builtins",
         "https://github.com/rust-lang/compiler-builtins.git",
-        "0.1.50",
+        "0.1.66",
     );
     apply_patches("compiler-builtins", Path::new("build_sysroot/compiler-builtins"));
 }
diff --git a/compiler/rustc_codegen_cranelift/example/issue-91827-extern-types.rs b/compiler/rustc_codegen_cranelift/example/issue-91827-extern-types.rs
new file mode 100644
index 0000000000000..cf8fada5320f9
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/example/issue-91827-extern-types.rs
@@ -0,0 +1,60 @@
+// Copied from rustc ui test suite
+
+// run-pass
+//
+// Test that we can handle unsized types with an extern type tail part.
+// Regression test for issue #91827.
+
+#![feature(const_ptr_offset_from)]
+#![feature(const_slice_from_raw_parts)]
+#![feature(extern_types)]
+
+use std::ptr::addr_of;
+
+extern "C" {
+    type Opaque;
+}
+
+unsafe impl Sync for Opaque {}
+
+#[repr(C)]
+pub struct List<T> {
+    len: usize,
+    data: [T; 0],
+    tail: Opaque,
+}
+
+#[repr(C)]
+pub struct ListImpl<T, const N: usize> {
+    len: usize,
+    data: [T; N],
+}
+
+impl<T> List<T> {
+    const fn as_slice(&self) -> &[T] {
+        unsafe { std::slice::from_raw_parts(self.data.as_ptr(), self.len) }
+    }
+}
+
+impl<T, const N: usize> ListImpl<T, N> {
+    const fn as_list(&self) -> &List<T> {
+        unsafe { std::mem::transmute(self) }
+    }
+}
+
+pub static A: ListImpl<u128, 3> = ListImpl {
+    len: 3,
+    data: [5, 6, 7],
+};
+pub static A_REF: &'static List<u128> = A.as_list();
+pub static A_TAIL_OFFSET: isize = tail_offset(A.as_list());
+
+const fn tail_offset<T>(list: &List<T>) -> isize {
+    unsafe { (addr_of!(list.tail) as *const u8).offset_from(list as *const List<T> as *const u8) }
+}
+
+fn main() {
+    assert_eq!(A_REF.as_slice(), &[5, 6, 7]);
+    // Check that interpreter and code generation agree about the position of the tail field.
+    assert_eq!(A_TAIL_OFFSET, tail_offset(A_REF));
+}
diff --git a/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-unsupported-tests.patch b/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-unsupported-tests.patch
index 2e68369466363..c13259086917b 100644
--- a/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-unsupported-tests.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-unsupported-tests.patch
@@ -1,41 +1,20 @@
-From 6bfce5dc2cbf834c74dbccb7538adc08c6eb57e7 Mon Sep 17 00:00:00 2001
+From 97c473937382a5b5858d9cce3c947855d23b2dc5 Mon Sep 17 00:00:00 2001
 From: bjorn3 <bjorn3@users.noreply.github.com>
-Date: Sun, 25 Jul 2021 18:39:31 +0200
+Date: Thu, 18 Nov 2021 19:28:40 +0100
 Subject: [PATCH] Disable unsupported tests
 
 ---
- crates/core_simd/src/vector.rs        |  2 ++
- crates/core_simd/src/math.rs         |  4 ++++
- crates/core_simd/tests/masks.rs      | 12 ------------
- crates/core_simd/tests/ops_macros.rs |  6 ++++++
- crates/core_simd/tests/round.rs      |  2 ++
- 6 files changed, 15 insertions(+), 13 deletions(-)
+ crates/core_simd/src/math.rs         | 6 ++++++
+ crates/core_simd/src/vector.rs       | 2 ++
+ crates/core_simd/tests/masks.rs      | 2 ++
+ crates/core_simd/tests/ops_macros.rs | 4 ++++
+ 4 files changed, 14 insertions(+)
 
-diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs
-index 25c5309..2b3d819 100644
---- a/crates/core_simd/src/vector.rs
-+++ b/crates/core_simd/src/vector.rs
-@@ -22,6 +22,7 @@ where
-         self.0
-     }
- 
-+    /*
-     /// SIMD gather: construct a SIMD vector by reading from a slice, using potentially discontiguous indices.
-     /// If an index is out of bounds, that lane instead selects the value from the "or" vector.
-     /// ```
-@@ -150,6 +151,7 @@ where
-             // Cleared ☢️ *mut T Zone
-         }
-     }
-+    */
- }
- 
- impl<T, const LANES: usize> Copy for Simd<T, LANES>
 diff --git a/crates/core_simd/src/math.rs b/crates/core_simd/src/math.rs
-index 7290a28..e394730 100644
+index 2bae414..2f87499 100644
 --- a/crates/core_simd/src/math.rs
 +++ b/crates/core_simd/src/math.rs
-@@ -2,6 +2,7 @@ macro_rules! impl_uint_arith {
+@@ -5,6 +5,7 @@ macro_rules! impl_uint_arith {
      ($($ty:ty),+) => {
          $( impl<const LANES: usize> Simd<$ty, LANES> where LaneCount<LANES>: SupportedLaneCount {
  
@@ -43,15 +22,15 @@ index 7290a28..e394730 100644
              /// Lanewise saturating add.
              ///
              /// # Examples
-@@ -38,6 +39,7 @@ macro_rules! impl_uint_arith {
+@@ -43,6 +44,7 @@ macro_rules! impl_uint_arith {
              pub fn saturating_sub(self, second: Self) -> Self {
-                 unsafe { crate::intrinsics::simd_saturating_sub(self, second) }
+                 unsafe { simd_saturating_sub(self, second) }
              }
 +            */
          })+
      }
  }
-@@ -46,6 +48,7 @@ macro_rules! impl_int_arith {
+@@ -51,6 +53,7 @@ macro_rules! impl_int_arith {
      ($($ty:ty),+) => {
          $( impl<const LANES: usize> Simd<$ty, LANES> where LaneCount<LANES>: SupportedLaneCount {
  
@@ -59,7 +38,23 @@ index 7290a28..e394730 100644
              /// Lanewise saturating add.
              ///
              /// # Examples
-@@ -141,6 +144,7 @@ macro_rules! impl_int_arith {
+@@ -89,6 +92,7 @@ macro_rules! impl_int_arith {
+             pub fn saturating_sub(self, second: Self) -> Self {
+                 unsafe { simd_saturating_sub(self, second) }
+             }
++            */
+ 
+             /// Lanewise absolute value, implemented in Rust.
+             /// Every lane becomes its absolute value.
+@@ -109,6 +113,7 @@ macro_rules! impl_int_arith {
+                 (self^m) - m
+             }
+ 
++            /*
+             /// Lanewise saturating absolute value, implemented in Rust.
+             /// As abs(), except the MIN value becomes MAX instead of itself.
+             ///
+@@ -151,6 +156,7 @@ macro_rules! impl_int_arith {
              pub fn saturating_neg(self) -> Self {
                  Self::splat(0).saturating_sub(self)
              }
@@ -67,51 +62,51 @@ index 7290a28..e394730 100644
          })+
      }
  }
+diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs
+index 7c5ec2b..c8631e8 100644
+--- a/crates/core_simd/src/vector.rs
++++ b/crates/core_simd/src/vector.rs
+@@ -75,6 +75,7 @@ where
+         Self(array)
+     }
+ 
++    /*
+     /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
+     /// If an index is out-of-bounds, the lane is instead selected from the `or` vector.
+     ///
+@@ -297,6 +298,7 @@ where
+             // Cleared ☢️ *mut T Zone
+         }
+     }
++    */
+ }
+ 
+ impl<T, const LANES: usize> Copy for Simd<T, LANES>
 diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs
-index 61d8e44..2bccae2 100644
+index 6a8ecd3..68fcb49 100644
 --- a/crates/core_simd/tests/masks.rs
 +++ b/crates/core_simd/tests/masks.rs
-@@ -67,19 +67,6 @@ macro_rules! test_mask_api {
-                 assert_eq!(int.to_array(), [-1, 0, 0, -1, 0, 0, -1, 0]);
+@@ -68,6 +68,7 @@ macro_rules! test_mask_api {
                  assert_eq!(core_simd::Mask::<$type, 8>::from_int(int), mask);
              }
--
--            #[cfg(feature = "generic_const_exprs")]
--            #[test]
--            fn roundtrip_bitmask_conversion() {
--                let values = [
--                    true, false, false, true, false, false, true, false,
--                    true, true, false, false, false, false, false, true,
--                ];
--                let mask = core_simd::Mask::<$type, 16>::from_array(values);
--                let bitmask = mask.to_bitmask();
--                assert_eq!(bitmask, [0b01001001, 0b10000011]);
--                assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask(bitmask), mask);
--            }
+ 
++            /*
+             #[cfg(feature = "generic_const_exprs")]
+             #[test]
+             fn roundtrip_bitmask_conversion() {
+@@ -80,6 +81,7 @@ macro_rules! test_mask_api {
+                 assert_eq!(bitmask, [0b01001001, 0b10000011]);
+                 assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask(bitmask), mask);
+             }
++            */
          }
      }
  }
 diff --git a/crates/core_simd/tests/ops_macros.rs b/crates/core_simd/tests/ops_macros.rs
-index cb39e73..fc0ebe1 100644
+index 31b7ee2..bd04b3c 100644
 --- a/crates/core_simd/tests/ops_macros.rs
 +++ b/crates/core_simd/tests/ops_macros.rs
-@@ -435,6 +435,7 @@ macro_rules! impl_float_tests {
-                     )
-                 }
- 
-+                /*
-                 fn mul_add<const LANES: usize>() {
-                     test_helpers::test_ternary_elementwise(
-                         &Vector::<LANES>::mul_add,
-@@ -442,6 +443,7 @@ macro_rules! impl_float_tests {
-                         &|_, _, _| true,
-                     )
-                 }
-+                */
- 
-                 fn recip<const LANES: usize>() {
-                     test_helpers::test_unary_elementwise(
-@@ -581,6 +585,7 @@ macro_rules! impl_float_tests {
+@@ -567,6 +567,7 @@ macro_rules! impl_float_tests {
                      });
                  }
  
@@ -119,7 +114,7 @@ index cb39e73..fc0ebe1 100644
                  fn horizontal_max<const LANES: usize>() {
                      test_helpers::test_1(&|x| {
                          let vmax = Vector::<LANES>::from_array(x).horizontal_max();
-@@ -604,6 +609,7 @@ macro_rules! impl_float_tests {
+@@ -590,6 +591,7 @@ macro_rules! impl_float_tests {
                          Ok(())
                      });
                  }
@@ -127,26 +122,22 @@ index cb39e73..fc0ebe1 100644
              }
  
              #[cfg(feature = "std")]
-diff --git a/crates/core_simd/tests/round.rs b/crates/core_simd/tests/round.rs
-index 37044a7..4cdc6b7 100644
---- a/crates/core_simd/tests/round.rs
-+++ b/crates/core_simd/tests/round.rs
-@@ -25,6 +25,7 @@ macro_rules! float_rounding_test {
-                     )
-                 }
+@@ -604,6 +606,7 @@ macro_rules! impl_float_tests {
+                         )
+                     }
  
-+                /*
-                 fn round<const LANES: usize>() {
-                     test_helpers::test_unary_elementwise(
-                         &Vector::<LANES>::round,
-@@ -32,6 +33,7 @@ macro_rules! float_rounding_test {
-                         &|_| true,
-                     )
++                    /*
+                     fn mul_add<const LANES: usize>() {
+                         test_helpers::test_ternary_elementwise(
+                             &Vector::<LANES>::mul_add,
+@@ -611,6 +614,7 @@ macro_rules! impl_float_tests {
+                             &|_, _, _| true,
+                         )
+                     }
++                    */
                  }
-+                */
- 
-                 fn trunc<const LANES: usize>() {
-                     test_helpers::test_unary_elementwise(
+             }
+         }
 -- 
 2.26.2.7.g19db9cfb68
 
diff --git a/compiler/rustc_codegen_cranelift/patches/0027-sysroot-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-sysroot-128bit-atomic-operations.patch
index e2d07bd126702..ffee641457ab2 100644
--- a/compiler/rustc_codegen_cranelift/patches/0027-sysroot-128bit-atomic-operations.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0027-sysroot-128bit-atomic-operations.patch
@@ -107,7 +107,7 @@ index fa96b7a..2854f9c 100644
      inner::monotonize(raw)
  }
  
--#[cfg(all(target_has_atomic = "64", not(target_has_atomic = "128")))]
+-#[cfg(any(all(target_has_atomic = "64", not(target_has_atomic = "128")), target_arch = "aarch64"))]
 +#[cfg(target_has_atomic = "64")]
  pub mod inner {
      use crate::sync::atomic::AtomicU64;
@@ -117,7 +117,7 @@ index fa96b7a..2854f9c 100644
  }
  
 +/*
- #[cfg(target_has_atomic = "128")]
+ #[cfg(all(target_has_atomic = "128", not(target_arch = "aarch64")))]
  pub mod inner {
      use crate::sync::atomic::AtomicU128;
 @@ -94,8 +95,9 @@ pub mod inner {
diff --git a/compiler/rustc_codegen_cranelift/patches/0028-sysroot-Disable-long-running-tests.patch b/compiler/rustc_codegen_cranelift/patches/0028-sysroot-Disable-long-running-tests.patch
new file mode 100644
index 0000000000000..bf74a74c7c4b8
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/patches/0028-sysroot-Disable-long-running-tests.patch
@@ -0,0 +1,30 @@
+From 0ffdd8eda8df364391c8ac6e1ce92c73ba9254d4 Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Fri, 3 Dec 2021 12:16:30 +0100
+Subject: [PATCH] Disable long running tests
+
+---
+ library/core/tests/slice.rs | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/library/core/tests/slice.rs b/library/core/tests/slice.rs
+index 2c8f00a..44847ee 100644
+--- a/library/core/tests/slice.rs
++++ b/library/core/tests/slice.rs
+@@ -2332,7 +2332,8 @@ macro_rules! empty_max_mut {
+     };
+ }
+ 
++/*
+ #[cfg(not(miri))] // Comparing usize::MAX many elements takes forever in Miri (and in rustc without optimizations)
+ take_tests! {
+     slice: &[(); usize::MAX], method: take,
+     (take_in_bounds_max_range_to, (..usize::MAX), Some(EMPTY_MAX), &[(); 0]),
+@@ -2345,3 +2347,4 @@ take_tests! {
+     (take_mut_oob_max_range_to_inclusive, (..=usize::MAX), None, empty_max_mut!()),
+     (take_mut_in_bounds_max_range_from, (usize::MAX..), Some(&mut [] as _), empty_max_mut!()),
+ }
++*/
+-- 
+2.26.2.7.g19db9cfb68
+
diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain
index 360570b3ae7a1..7b5db307a2dc2 100644
--- a/compiler/rustc_codegen_cranelift/rust-toolchain
+++ b/compiler/rustc_codegen_cranelift/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2021-09-19"
+channel = "nightly-2021-12-20"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
index ca83e7096b86d..46c3b5b7f11ad 100644
--- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
@@ -33,7 +33,7 @@ index d95b5b7f17f..00b6f0e3635 100644
  [dependencies]
  core = { path = "../core" }
 -compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std'] }
-+compiler_builtins = { version = "0.1.46", features = ['rustc-dep-of-std', 'no-asm'] }
++compiler_builtins = { version = "0.1.66", features = ['rustc-dep-of-std', 'no-asm'] }
 
  [dev-dependencies]
  rand = "0.7"
@@ -53,5 +53,6 @@ local-rebuild = true
 [rust]
 codegen-backends = ["cranelift"]
 deny-warnings = false
+verbose-tests = false
 EOF
 popd
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
index b714d47fec2a6..99fddf5361e43 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -10,7 +10,7 @@ pushd rust
 
 cargo install ripgrep
 
-rm -r src/test/ui/{extern/,panics/,unsized-locals/,thinlto/,simd*,*lto*.rs,linkage*,unwind-*.rs} || true
+rm -r src/test/ui/{extern/,panics/,unsized-locals/,lto/,simd*,linkage*,unwind-*.rs} || true
 for test in $(rg --files-with-matches "asm!|catch_unwind|should_panic|lto|// needs-asm-support" src/test/ui); do
   rm $test
 done
@@ -30,19 +30,20 @@ rm src/test/ui/cleanup-rvalue-temp-during-incomplete-alloc.rs
 rm src/test/ui/issues/issue-26655.rs
 rm src/test/ui/issues/issue-29485.rs
 rm src/test/ui/issues/issue-30018-panic.rs
-rm src/test/ui/multi-panic.rs
+rm src/test/ui/process/multi-panic.rs
 rm src/test/ui/sepcomp/sepcomp-unwind.rs
 rm src/test/ui/structs-enums/unit-like-struct-drop-run.rs
-rm src/test/ui/terminate-in-initializer.rs
+rm src/test/ui/drop/terminate-in-initializer.rs
 rm src/test/ui/threads-sendsync/task-stderr.rs
 rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs
 rm src/test/ui/drop/drop-trait-enum.rs
 rm src/test/ui/numbers-arithmetic/issue-8460.rs
-rm src/test/ui/rt-explody-panic-payloads.rs
+rm src/test/ui/runtime/rt-explody-panic-payloads.rs
 rm src/test/incremental/change_crate_dep_kind.rs
+rm src/test/ui/threads-sendsync/unwind-resource.rs
 
 rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
-rm src/test/ui/init-large-type.rs # same
+rm src/test/ui/codegen/init-large-type.rs # same
 rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected
 rm src/test/ui/issues/issue-33992.rs # unsupported linkages
 rm src/test/ui/issues/issue-51947.rs # same
@@ -65,6 +66,7 @@ rm src/test/incremental/lto.rs # requires lto
 
 rm -r src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/
 rm -r src/test/run-make/unstable-flag-required # same
+rm -r src/test/run-make/rustdoc-* # same
 rm -r src/test/run-make/emit-named-files # requires full --emit support
 
 rm src/test/pretty/asm.rs # inline asm
@@ -74,7 +76,10 @@ rm -r src/test/run-pass-valgrind/unsized-locals
 
 rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
 rm src/test/ui/json-bom-plus-crlf.rs # same
+rm src/test/ui/intrinsics/const-eval-select-x86_64.rs # same
 rm src/test/ui/match/issue-82392.rs # differing error
+rm src/test/ui/consts/min_const_fn/address_of_const.rs # same
+rm src/test/ui/consts/issue-miri-1910.rs # same
 rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
 
 rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition
@@ -88,6 +93,10 @@ rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization
 rm src/test/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
 rm src/test/ui/abi/variadic-ffi.rs # requires callee side vararg support
 
+rm src/test/ui/command/command-current-dir.rs # can't find libstd.so
+
+rm src/test/ui/abi/stack-protector.rs # requires stack protector support
+
 echo "[TEST] rustc test suite"
 RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui}
 popd
diff --git a/compiler/rustc_codegen_cranelift/scripts/tests.sh b/compiler/rustc_codegen_cranelift/scripts/tests.sh
index 28a7980d6613c..fd2b3761ff036 100755
--- a/compiler/rustc_codegen_cranelift/scripts/tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/tests.sh
@@ -35,6 +35,10 @@ function base_sysroot_tests() {
     $MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
 
+    echo "[AOT] issue_91827_extern_types"
+    $MY_RUSTC example/issue-91827-extern-types.rs --crate-name issue_91827_extern_types --crate-type bin --target "$TARGET_TRIPLE"
+    $RUN_WRAPPER ./target/out/issue_91827_extern_types
+
     echo "[AOT] alloc_system"
     $MY_RUSTC example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
 
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 78fdf9c02d06a..72ebc84c1a344 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -18,11 +18,11 @@ pub(crate) use self::returning::codegen_return;
 
 fn clif_sig_from_fn_abi<'tcx>(
     tcx: TyCtxt<'tcx>,
-    triple: &target_lexicon::Triple,
+    default_call_conv: CallConv,
     fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
 ) -> Signature {
     let call_conv = match fn_abi.conv {
-        Conv::Rust | Conv::C => CallConv::triple_default(triple),
+        Conv::Rust | Conv::C => default_call_conv,
         Conv::X86_64SysV => CallConv::SystemV,
         Conv::X86_64Win64 => CallConv::WindowsFastcall,
         Conv::ArmAapcs
@@ -55,7 +55,7 @@ pub(crate) fn get_function_sig<'tcx>(
     assert!(!inst.substs.needs_infer());
     clif_sig_from_fn_abi(
         tcx,
-        triple,
+        CallConv::triple_default(triple),
         &RevealAllLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()),
     )
 }
@@ -91,7 +91,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         returns: Vec<AbiParam>,
         args: &[Value],
     ) -> &[Value] {
-        let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) };
+        let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
         let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
         let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
         let call_inst = self.bcx.ins().call(func_ref, args);
@@ -420,7 +420,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
             }
 
             let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0].value, idx);
-            let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi);
+            let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
             let sig = fx.bcx.import_signature(sig);
 
             (CallTarget::Indirect(sig, method), Some(ptr))
@@ -440,7 +440,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
             }
 
             let func = codegen_operand(fx, func).load_scalar(fx);
-            let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi);
+            let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
             let sig = fx.bcx.import_signature(sig);
 
             (CallTarget::Indirect(sig, func), None)
@@ -531,7 +531,7 @@ pub(crate) fn codegen_drop<'tcx>(
                 let fn_abi =
                     RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty());
 
-                let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi);
+                let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
                 let sig = fx.bcx.import_signature(sig);
                 fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]);
             }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index 45d4906259312..9f0bd31e95fcc 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -117,7 +117,9 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             PassMode::Cast(cast) => cast_target_to_abi_params(cast),
             PassMode::Indirect { attrs, extra_attrs: None, on_stack } => {
                 if on_stack {
-                    let size = u32::try_from(self.layout.size.bytes()).unwrap();
+                    // Abi requires aligning struct size to pointer size
+                    let size = self.layout.size.align_to(tcx.data_layout.pointer_align.abi);
+                    let size = u32::try_from(size.bytes()).unwrap();
                     smallvec![apply_arg_attrs_to_abi_param(
                         AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),),
                         attrs
@@ -204,7 +206,6 @@ pub(super) fn from_casted_value<'tcx>(
         // It may also be smaller for example when the type is a wrapper around an integer with a
         // larger alignment than the integer.
         size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
-        offset: None,
     });
     let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
     let mut offset = 0;
diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs
index 71f510c037f11..b0eb3864d80c8 100644
--- a/compiler/rustc_codegen_cranelift/src/archive.rs
+++ b/compiler/rustc_codegen_cranelift/src/archive.rs
@@ -1,7 +1,6 @@
 //! Creation of ar archives like for the lib and staticlib crate type
 
 use std::collections::BTreeMap;
-use std::convert::TryFrom;
 use std::fs::File;
 use std::io::{self, Read, Seek};
 use std::path::{Path, PathBuf};
@@ -10,7 +9,7 @@ use rustc_codegen_ssa::back::archive::ArchiveBuilder;
 use rustc_session::Session;
 
 use object::read::archive::ArchiveFile;
-use object::{Object, ObjectSymbol, ReadCache, SymbolKind};
+use object::{Object, ObjectSymbol, ReadCache};
 
 #[derive(Debug)]
 enum ArchiveEntry {
@@ -150,12 +149,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
                             object
                                 .symbols()
                                 .filter_map(|symbol| {
-                                    if symbol.is_undefined()
-                                        || symbol.is_local()
-                                        || symbol.kind() != SymbolKind::Data
-                                            && symbol.kind() != SymbolKind::Text
-                                            && symbol.kind() != SymbolKind::Tls
-                                    {
+                                    if symbol.is_undefined() || symbol.is_local() {
                                         None
                                     } else {
                                         symbol.name().map(|name| name.as_bytes().to_vec()).ok()
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 371c71de62fbb..fc2f04f146e9d 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -49,13 +49,15 @@ pub(crate) fn codegen_fn<'tcx>(
         (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
 
     // Make FunctionCx
-    let pointer_type = module.target_config().pointer_type();
+    let target_config = module.target_config();
+    let pointer_type = target_config.pointer_type();
     let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
 
     let mut fx = FunctionCx {
         cx,
         module,
         tcx,
+        target_config,
         pointer_type,
         constants_cx: ConstantCx::new(),
 
@@ -72,8 +74,6 @@ pub(crate) fn codegen_fn<'tcx>(
         clif_comments,
         source_info_set: indexmap::IndexSet::new(),
         next_ssa_var: 0,
-
-        inline_asm_index: 0,
     };
 
     let arg_uninhabited = fx
@@ -204,7 +204,6 @@ pub(crate) fn verify_func(
                 tcx.sess.err(&format!("{:?}", err));
                 let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error(
                     &func,
-                    None,
                     Some(Box::new(writer)),
                     err,
                 );
@@ -296,9 +295,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                     AssertKind::BoundsCheck { ref len, ref index } => {
                         let len = codegen_operand(fx, len).load_scalar(fx);
                         let index = codegen_operand(fx, index).load_scalar(fx);
-                        let location = fx
-                            .get_caller_location(source_info.span)
-                            .load_scalar(fx);
+                        let location = fx.get_caller_location(source_info.span).load_scalar(fx);
 
                         codegen_panic_inner(
                             fx,
@@ -681,7 +678,7 @@ fn codegen_stmt<'tcx>(
                         // FIXME use emit_small_memset where possible
                         let addr = lval.to_ptr().get_addr(fx);
                         let val = operand.load_scalar(fx);
-                        fx.bcx.call_memset(fx.module.target_config(), addr, val, times);
+                        fx.bcx.call_memset(fx.target_config, addr, val, times);
                     } else {
                         let loop_block = fx.bcx.create_block();
                         let loop_block2 = fx.bcx.create_block();
@@ -754,8 +751,7 @@ fn codegen_stmt<'tcx>(
                         NullOp::AlignOf => layout.align.abi.bytes(),
                         NullOp::Box => unreachable!(),
                     };
-                    let val =
-                        CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());
+                    let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());
                     lval.write_cvalue(fx, val);
                 }
                 Rvalue::Aggregate(ref kind, ref operands) => match kind.as_ref() {
@@ -803,7 +799,7 @@ fn codegen_stmt<'tcx>(
             let elem_size: u64 = pointee.size.bytes();
             let bytes =
                 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
-            fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes);
+            fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
         }
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index 0e84681d9ad94..644204d10b8ed 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -1,3 +1,4 @@
+use cranelift_codegen::isa::TargetFrontendConfig;
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
@@ -235,6 +236,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>,
     pub(crate) module: &'m mut dyn Module,
     pub(crate) tcx: TyCtxt<'tcx>,
+    pub(crate) target_config: TargetFrontendConfig, // Cached from module
     pub(crate) pointer_type: Type, // Cached from module
     pub(crate) constants_cx: ConstantCx,
 
@@ -255,8 +257,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
 
     /// This should only be accessed by `CPlace::new_var`.
     pub(crate) next_ssa_var: u32,
-
-    pub(crate) inline_asm_index: u32,
 }
 
 impl<'tcx> LayoutOfHelpers<'tcx> for FunctionCx<'_, '_, 'tcx> {
@@ -359,10 +359,6 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         crate::constant::codegen_const_value(self, const_loc, self.tcx.caller_location_ty())
     }
 
-    pub(crate) fn triple(&self) -> &target_lexicon::Triple {
-        self.module.isa().triple()
-    }
-
     pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value {
         let mut data_ctx = DataContext::new();
         data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice());
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
index c8c2d50b03409..4120ba6e53352 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
@@ -69,8 +69,6 @@ impl WriterRelocate {
     /// Perform the collected relocations to be usable for JIT usage.
     #[cfg(feature = "jit")]
     pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec<u8> {
-        use std::convert::TryInto;
-
         for reloc in self.relocs.drain(..) {
             match reloc.name {
                 super::DebugRelocName::Section(_) => unreachable!(),
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
index 6d172817cb12e..dd19dd5d2b91c 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
@@ -10,7 +10,7 @@ use crate::prelude::*;
 use rustc_index::vec::IndexVec;
 
 use cranelift_codegen::entity::EntityRef;
-use cranelift_codegen::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLoc};
+use cranelift_codegen::ir::{LabelValueLoc, ValueLabel};
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::ValueLocRange;
 
@@ -67,7 +67,12 @@ impl<'tcx> DebugContext<'tcx> {
             rustc_interface::util::version_str().unwrap_or("unknown version"),
             cranelift_codegen::VERSION,
         );
-        let comp_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped).into_owned();
+        let comp_dir = tcx
+            .sess
+            .opts
+            .working_dir
+            .to_string_lossy(FileNameDisplayPreference::Remapped)
+            .into_owned();
         let (name, file_info) = match tcx.sess.local_crate_source_file.clone() {
             Some(path) => {
                 let name = path.to_string_lossy().into_owned();
@@ -250,7 +255,7 @@ impl<'tcx> DebugContext<'tcx> {
 
         // FIXME make it more reliable and implement scopes before re-enabling this.
         if false {
-            let value_labels_ranges = context.build_value_labels_ranges(isa).unwrap();
+            let value_labels_ranges = std::collections::HashMap::new(); // FIXME
 
             for (local, _local_decl) in mir.local_decls.iter_enumerated() {
                 let ty = self.tcx.subst_and_normalize_erasing_regions(
@@ -264,7 +269,6 @@ impl<'tcx> DebugContext<'tcx> {
                     self,
                     isa,
                     symbol,
-                    context,
                     &local_map,
                     &value_labels_ranges,
                     Place { local, projection: ty::List::empty() },
@@ -283,7 +287,6 @@ fn place_location<'tcx>(
     debug_context: &mut DebugContext<'tcx>,
     isa: &dyn TargetIsa,
     symbol: usize,
-    context: &Context,
     local_map: &IndexVec<mir::Local, CPlace<'tcx>>,
     #[allow(rustc::default_hash_types)] value_labels_ranges: &std::collections::HashMap<
         ValueLabel,
@@ -306,12 +309,7 @@ fn place_location<'tcx>(
                                 addend: i64::from(value_loc_range.start),
                             },
                             end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) },
-                            data: translate_loc(
-                                isa,
-                                value_loc_range.loc,
-                                &context.func.stack_slots,
-                            )
-                            .unwrap(),
+                            data: translate_loc(isa, value_loc_range.loc).unwrap(),
                         })
                         .collect(),
                 );
@@ -340,34 +338,14 @@ fn place_location<'tcx>(
             AttributeValue::Exprloc(Expression::new())
 
             // For PointerBase::Stack:
-            //AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot), &context.func.stack_slots).unwrap())
+            //AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot)).unwrap())
         }
     }
 }
 
 // Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137
-fn translate_loc(
-    isa: &dyn TargetIsa,
-    loc: LabelValueLoc,
-    stack_slots: &StackSlots,
-) -> Option<Expression> {
+fn translate_loc(isa: &dyn TargetIsa, loc: LabelValueLoc) -> Option<Expression> {
     match loc {
-        LabelValueLoc::ValueLoc(ValueLoc::Reg(reg)) => {
-            let machine_reg = isa.map_dwarf_register(reg).unwrap();
-            let mut expr = Expression::new();
-            expr.op_reg(gimli::Register(machine_reg));
-            Some(expr)
-        }
-        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
-            if let Some(ss_offset) = stack_slots[ss].offset {
-                let mut expr = Expression::new();
-                expr.op_breg(X86_64::RBP, i64::from(ss_offset) + 16);
-                Some(expr)
-            } else {
-                None
-            }
-        }
-        LabelValueLoc::ValueLoc(ValueLoc::Unassigned) => unreachable!(),
         LabelValueLoc::Reg(reg) => {
             let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap();
             let mut expr = Expression::new();
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs
index 9984dc92c44d0..9dc9b2cf9f8ad 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs
@@ -1,5 +1,3 @@
-use std::convert::{TryFrom, TryInto};
-
 use rustc_data_structures::fx::FxHashMap;
 
 use cranelift_module::FuncId;
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index c09be5f75979f..7f888c80464d4 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -4,6 +4,7 @@
 use std::path::PathBuf;
 
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
 use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_metadata::EncodedMetadata;
@@ -123,6 +124,7 @@ fn module_codegen(
         backend_config.clone(),
         module.isa(),
         tcx.sess.opts.debuginfo != DebugInfo::None,
+        cgu_name,
     );
     super::predefine_mono_items(tcx, &mut module, &mono_items);
     for (mono_item, _) in mono_items {
@@ -277,7 +279,8 @@ pub(crate) fn run_aot(
             let tmp_file =
                 tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name));
 
-            let obj = crate::metadata::new_metadata_object(tcx, &metadata_cgu_name, &metadata);
+            let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx);
+            let obj = create_compressed_metadata_file(tcx.sess, &metadata, &symbol_name);
 
             if let Err(err) = std::fs::write(&tmp_file, obj) {
                 tcx.sess.fatal(&format!("error writing metadata object file: {}", err));
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index 76fbc9ad51e8e..309d27090b5cb 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -3,7 +3,7 @@
 
 use std::cell::RefCell;
 use std::ffi::CString;
-use std::lazy::{Lazy, SyncOnceCell};
+use std::lazy::SyncOnceCell;
 use std::os::raw::{c_char, c_int};
 use std::sync::{mpsc, Mutex};
 
@@ -11,6 +11,7 @@ use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
 use rustc_session::Session;
+use rustc_span::Symbol;
 
 use cranelift_jit::{JITBuilder, JITModule};
 
@@ -23,7 +24,7 @@ struct JitState {
 }
 
 thread_local! {
-    static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None);
+    static LAZY_JIT_STATE: RefCell<Option<JitState>> = const { RefCell::new(None) };
 }
 
 /// The Sender owned by the rustc thread
@@ -50,12 +51,11 @@ impl UnsafeMessage {
     fn send(self) -> Result<(), mpsc::SendError<UnsafeMessage>> {
         thread_local! {
             /// The Sender owned by the local thread
-            static LOCAL_MESSAGE_SENDER: Lazy<mpsc::Sender<UnsafeMessage>> = Lazy::new(||
+            static LOCAL_MESSAGE_SENDER: mpsc::Sender<UnsafeMessage> =
                 GLOBAL_MESSAGE_SENDER
                     .get().unwrap()
                     .lock().unwrap()
-                    .clone()
-            );
+                    .clone();
         }
         LOCAL_MESSAGE_SENDER.with(|sender| sender.send(self))
     }
@@ -76,7 +76,13 @@ fn create_jit_module<'tcx>(
     jit_builder.symbols(imported_symbols);
     let mut jit_module = JITModule::new(jit_builder);
 
-    let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false);
+    let mut cx = crate::CodegenCx::new(
+        tcx,
+        backend_config.clone(),
+        jit_module.isa(),
+        false,
+        Symbol::intern("dummy_cgu_name"),
+    );
 
     crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context);
     crate::main_shim::maybe_create_entry_wrapper(
@@ -246,7 +252,13 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) ->
 
             jit_module.prepare_for_function_redefine(func_id).unwrap();
 
-            let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false);
+            let mut cx = crate::CodegenCx::new(
+                tcx,
+                backend_config,
+                jit_module.isa(),
+                false,
+                Symbol::intern("dummy_cgu_name"),
+            );
             tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
 
             assert!(cx.global_asm.is_empty());
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index f5c9b0b5480a5..93384bc551101 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -6,6 +6,7 @@ use std::fmt::Write;
 
 use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::mir::InlineAsmOperand;
+use rustc_span::Symbol;
 use rustc_target::asm::*;
 
 pub(crate) fn codegen_inline_asm<'tcx>(
@@ -17,10 +18,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
 ) {
     // FIXME add .eh_frame unwind info directives
 
-    if template.is_empty() {
-        // Black box
-        return;
-    } else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
+    if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
         let true_ = fx.bcx.ins().iconst(types::I32, 1);
         fx.bcx.ins().trapnz(true_, TrapCode::User(1));
         return;
@@ -41,8 +39,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         assert_eq!(operands.len(), 4);
         let (leaf, eax_place) = match operands[1] {
             InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
-                let reg = expect_reg(reg);
-                assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax));
+                assert_eq!(
+                    reg,
+                    InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax))
+                );
                 (
                     crate::base::codegen_operand(fx, in_value).load_scalar(fx),
                     crate::base::codegen_place(fx, out_place.unwrap()),
@@ -64,8 +64,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         };
         let (sub_leaf, ecx_place) = match operands[2] {
             InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
-                let reg = expect_reg(reg);
-                assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::cx));
+                assert_eq!(
+                    reg,
+                    InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx))
+                );
                 (
                     crate::base::codegen_operand(fx, in_value).load_scalar(fx),
                     crate::base::codegen_place(fx, out_place.unwrap()),
@@ -75,8 +77,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         };
         let edx_place = match operands[3] {
             InlineAsmOperand::Out { reg, late: true, place } => {
-                let reg = expect_reg(reg);
-                assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::dx));
+                assert_eq!(
+                    reg,
+                    InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx))
+                );
                 crate::base::codegen_place(fx, place.unwrap())
             }
             _ => unreachable!(),
@@ -96,60 +100,59 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         crate::trap::trap_unimplemented(fx, "Alloca is not supported");
     }
 
-    let mut slot_size = Size::from_bytes(0);
-    let mut clobbered_regs = Vec::new();
     let mut inputs = Vec::new();
     let mut outputs = Vec::new();
 
-    let mut new_slot = |reg_class: InlineAsmRegClass| {
-        let reg_size = reg_class
-            .supported_types(InlineAsmArch::X86_64)
-            .iter()
-            .map(|(ty, _)| ty.size())
-            .max()
-            .unwrap();
-        let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap();
-        slot_size = slot_size.align_to(align);
-        let offset = slot_size;
-        slot_size += reg_size;
-        offset
+    let mut asm_gen = InlineAssemblyGenerator {
+        tcx: fx.tcx,
+        arch: fx.tcx.sess.asm_arch.unwrap(),
+        template,
+        operands,
+        options,
+        registers: Vec::new(),
+        stack_slots_clobber: Vec::new(),
+        stack_slots_input: Vec::new(),
+        stack_slots_output: Vec::new(),
+        stack_slot_size: Size::from_bytes(0),
     };
+    asm_gen.allocate_registers();
+    asm_gen.allocate_stack_slots();
+
+    let inline_asm_index = fx.cx.inline_asm_index.get();
+    fx.cx.inline_asm_index.set(inline_asm_index + 1);
+    let asm_name = format!(
+        "__inline_asm_{}_n{}",
+        fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
+        inline_asm_index
+    );
+
+    let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
+    fx.cx.global_asm.push_str(&generated_asm);
 
-    // FIXME overlap input and output slots to save stack space
-    for operand in operands {
+    for (i, operand) in operands.iter().enumerate() {
         match *operand {
-            InlineAsmOperand::In { reg, ref value } => {
-                let reg = expect_reg(reg);
-                clobbered_regs.push((reg, new_slot(reg.reg_class())));
+            InlineAsmOperand::In { reg: _, ref value } => {
                 inputs.push((
-                    reg,
-                    new_slot(reg.reg_class()),
+                    asm_gen.stack_slots_input[i].unwrap(),
                     crate::base::codegen_operand(fx, value).load_scalar(fx),
                 ));
             }
-            InlineAsmOperand::Out { reg, late: _, place } => {
-                let reg = expect_reg(reg);
-                clobbered_regs.push((reg, new_slot(reg.reg_class())));
+            InlineAsmOperand::Out { reg: _, late: _, place } => {
                 if let Some(place) = place {
                     outputs.push((
-                        reg,
-                        new_slot(reg.reg_class()),
+                        asm_gen.stack_slots_output[i].unwrap(),
                         crate::base::codegen_place(fx, place),
                     ));
                 }
             }
-            InlineAsmOperand::InOut { reg, late: _, ref in_value, out_place } => {
-                let reg = expect_reg(reg);
-                clobbered_regs.push((reg, new_slot(reg.reg_class())));
+            InlineAsmOperand::InOut { reg: _, late: _, ref in_value, out_place } => {
                 inputs.push((
-                    reg,
-                    new_slot(reg.reg_class()),
+                    asm_gen.stack_slots_input[i].unwrap(),
                     crate::base::codegen_operand(fx, in_value).load_scalar(fx),
                 ));
                 if let Some(out_place) = out_place {
                     outputs.push((
-                        reg,
-                        new_slot(reg.reg_class()),
+                        asm_gen.stack_slots_output[i].unwrap(),
                         crate::base::codegen_place(fx, out_place),
                     ));
                 }
@@ -160,110 +163,474 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         }
     }
 
-    let inline_asm_index = fx.inline_asm_index;
-    fx.inline_asm_index += 1;
-    let asm_name = format!("{}__inline_asm_{}", fx.symbol_name, inline_asm_index);
-
-    let generated_asm = generate_asm_wrapper(
-        &asm_name,
-        InlineAsmArch::X86_64,
-        options,
-        template,
-        clobbered_regs,
-        &inputs,
-        &outputs,
-    );
-    fx.cx.global_asm.push_str(&generated_asm);
-
-    call_inline_asm(fx, &asm_name, slot_size, inputs, outputs);
+    call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
 }
 
-fn generate_asm_wrapper(
-    asm_name: &str,
+struct InlineAssemblyGenerator<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
     arch: InlineAsmArch,
+    template: &'a [InlineAsmTemplatePiece],
+    operands: &'a [InlineAsmOperand<'tcx>],
     options: InlineAsmOptions,
-    template: &[InlineAsmTemplatePiece],
-    clobbered_regs: Vec<(InlineAsmReg, Size)>,
-    inputs: &[(InlineAsmReg, Size, Value)],
-    outputs: &[(InlineAsmReg, Size, CPlace<'_>)],
-) -> String {
-    let mut generated_asm = String::new();
-    writeln!(generated_asm, ".globl {}", asm_name).unwrap();
-    writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
-    writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
-    writeln!(generated_asm, "{}:", asm_name).unwrap();
-
-    generated_asm.push_str(".intel_syntax noprefix\n");
-    generated_asm.push_str("    push rbp\n");
-    generated_asm.push_str("    mov rbp,rdi\n");
-
-    // Save clobbered registers
-    if !options.contains(InlineAsmOptions::NORETURN) {
-        // FIXME skip registers saved by the calling convention
-        for &(reg, offset) in &clobbered_regs {
-            save_register(&mut generated_asm, arch, reg, offset);
+    registers: Vec<Option<InlineAsmReg>>,
+    stack_slots_clobber: Vec<Option<Size>>,
+    stack_slots_input: Vec<Option<Size>>,
+    stack_slots_output: Vec<Option<Size>>,
+    stack_slot_size: Size,
+}
+
+impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
+    fn allocate_registers(&mut self) {
+        let sess = self.tcx.sess;
+        let map = allocatable_registers(
+            self.arch,
+            |feature| sess.target_features.contains(&Symbol::intern(feature)),
+            &sess.target,
+        );
+        let mut allocated = FxHashMap::<_, (bool, bool)>::default();
+        let mut regs = vec![None; self.operands.len()];
+
+        // Add explicit registers to the allocated set.
+        for (i, operand) in self.operands.iter().enumerate() {
+            match *operand {
+                InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
+                    regs[i] = Some(reg);
+                    allocated.entry(reg).or_default().0 = true;
+                }
+                InlineAsmOperand::Out {
+                    reg: InlineAsmRegOrRegClass::Reg(reg), late: true, ..
+                } => {
+                    regs[i] = Some(reg);
+                    allocated.entry(reg).or_default().1 = true;
+                }
+                InlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
+                | InlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
+                    regs[i] = Some(reg);
+                    allocated.insert(reg, (true, true));
+                }
+                _ => (),
+            }
         }
-    }
 
-    // Write input registers
-    for &(reg, offset, _value) in inputs {
-        restore_register(&mut generated_asm, arch, reg, offset);
-    }
+        // Allocate out/inout/inlateout registers first because they are more constrained.
+        for (i, operand) in self.operands.iter().enumerate() {
+            match *operand {
+                InlineAsmOperand::Out {
+                    reg: InlineAsmRegOrRegClass::RegClass(class),
+                    late: false,
+                    ..
+                }
+                | InlineAsmOperand::InOut {
+                    reg: InlineAsmRegOrRegClass::RegClass(class), ..
+                } => {
+                    let mut alloc_reg = None;
+                    for &reg in &map[&class] {
+                        let mut used = false;
+                        reg.overlapping_regs(|r| {
+                            if allocated.contains_key(&r) {
+                                used = true;
+                            }
+                        });
+
+                        if !used {
+                            alloc_reg = Some(reg);
+                            break;
+                        }
+                    }
+
+                    let reg = alloc_reg.expect("cannot allocate registers");
+                    regs[i] = Some(reg);
+                    allocated.insert(reg, (true, true));
+                }
+                _ => (),
+            }
+        }
+
+        // Allocate in/lateout.
+        for (i, operand) in self.operands.iter().enumerate() {
+            match *operand {
+                InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
+                    let mut alloc_reg = None;
+                    for &reg in &map[&class] {
+                        let mut used = false;
+                        reg.overlapping_regs(|r| {
+                            if allocated.get(&r).copied().unwrap_or_default().0 {
+                                used = true;
+                            }
+                        });
+
+                        if !used {
+                            alloc_reg = Some(reg);
+                            break;
+                        }
+                    }
+
+                    let reg = alloc_reg.expect("cannot allocate registers");
+                    regs[i] = Some(reg);
+                    allocated.entry(reg).or_default().0 = true;
+                }
+                InlineAsmOperand::Out {
+                    reg: InlineAsmRegOrRegClass::RegClass(class),
+                    late: true,
+                    ..
+                } => {
+                    let mut alloc_reg = None;
+                    for &reg in &map[&class] {
+                        let mut used = false;
+                        reg.overlapping_regs(|r| {
+                            if allocated.get(&r).copied().unwrap_or_default().1 {
+                                used = true;
+                            }
+                        });
+
+                        if !used {
+                            alloc_reg = Some(reg);
+                            break;
+                        }
+                    }
+
+                    let reg = alloc_reg.expect("cannot allocate registers");
+                    regs[i] = Some(reg);
+                    allocated.entry(reg).or_default().1 = true;
+                }
+                _ => (),
+            }
+        }
 
-    if options.contains(InlineAsmOptions::ATT_SYNTAX) {
-        generated_asm.push_str(".att_syntax\n");
+        self.registers = regs;
     }
 
-    // The actual inline asm
-    for piece in template {
-        match piece {
-            InlineAsmTemplatePiece::String(s) => {
-                generated_asm.push_str(s);
+    fn allocate_stack_slots(&mut self) {
+        let mut slot_size = Size::from_bytes(0);
+        let mut slots_clobber = vec![None; self.operands.len()];
+        let mut slots_input = vec![None; self.operands.len()];
+        let mut slots_output = vec![None; self.operands.len()];
+
+        let new_slot_fn = |slot_size: &mut Size, reg_class: InlineAsmRegClass| {
+            let reg_size =
+                reg_class.supported_types(self.arch).iter().map(|(ty, _)| ty.size()).max().unwrap();
+            let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap();
+            let offset = slot_size.align_to(align);
+            *slot_size = offset + reg_size;
+            offset
+        };
+        let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
+
+        // Allocate stack slots for saving clobbered registers
+        let abi_clobber = InlineAsmClobberAbi::parse(
+            self.arch,
+            |feature| self.tcx.sess.target_features.contains(&Symbol::intern(feature)),
+            &self.tcx.sess.target,
+            Symbol::intern("C"),
+        )
+        .unwrap()
+        .clobbered_regs();
+        for (i, reg) in self.registers.iter().enumerate().filter_map(|(i, r)| r.map(|r| (i, r))) {
+            let mut need_save = true;
+            // If the register overlaps with a register clobbered by function call, then
+            // we don't need to save it.
+            for r in abi_clobber {
+                r.overlapping_regs(|r| {
+                    if r == reg {
+                        need_save = false;
+                    }
+                });
+
+                if !need_save {
+                    break;
+                }
+            }
+
+            if need_save {
+                slots_clobber[i] = Some(new_slot(reg.reg_class()));
             }
-            InlineAsmTemplatePiece::Placeholder { operand_idx: _, modifier: _, span: _ } => todo!(),
         }
+
+        // Allocate stack slots for inout
+        for (i, operand) in self.operands.iter().enumerate() {
+            match *operand {
+                InlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
+                    let slot = new_slot(reg.reg_class());
+                    slots_input[i] = Some(slot);
+                    slots_output[i] = Some(slot);
+                }
+                _ => (),
+            }
+        }
+
+        let slot_size_before_input = slot_size;
+        let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
+
+        // Allocate stack slots for input
+        for (i, operand) in self.operands.iter().enumerate() {
+            match *operand {
+                InlineAsmOperand::In { reg, .. }
+                | InlineAsmOperand::InOut { reg, out_place: None, .. } => {
+                    slots_input[i] = Some(new_slot(reg.reg_class()));
+                }
+                _ => (),
+            }
+        }
+
+        // Reset slot size to before input so that input and output operands can overlap
+        // and save some memory.
+        let slot_size_after_input = slot_size;
+        slot_size = slot_size_before_input;
+        let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
+
+        // Allocate stack slots for output
+        for (i, operand) in self.operands.iter().enumerate() {
+            match *operand {
+                InlineAsmOperand::Out { reg, place: Some(_), .. } => {
+                    slots_output[i] = Some(new_slot(reg.reg_class()));
+                }
+                _ => (),
+            }
+        }
+
+        slot_size = slot_size.max(slot_size_after_input);
+
+        self.stack_slots_clobber = slots_clobber;
+        self.stack_slots_input = slots_input;
+        self.stack_slots_output = slots_output;
+        self.stack_slot_size = slot_size;
     }
-    generated_asm.push('\n');
 
-    if options.contains(InlineAsmOptions::ATT_SYNTAX) {
-        generated_asm.push_str(".intel_syntax noprefix\n");
+    fn generate_asm_wrapper(&self, asm_name: &str) -> String {
+        let mut generated_asm = String::new();
+        writeln!(generated_asm, ".globl {}", asm_name).unwrap();
+        writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
+        writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
+        writeln!(generated_asm, "{}:", asm_name).unwrap();
+
+        let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);
+
+        if is_x86 {
+            generated_asm.push_str(".intel_syntax noprefix\n");
+        }
+        Self::prologue(&mut generated_asm, self.arch);
+
+        // Save clobbered registers
+        if !self.options.contains(InlineAsmOptions::NORETURN) {
+            for (reg, slot) in self
+                .registers
+                .iter()
+                .zip(self.stack_slots_clobber.iter().copied())
+                .filter_map(|(r, s)| r.zip(s))
+            {
+                Self::save_register(&mut generated_asm, self.arch, reg, slot);
+            }
+        }
+
+        // Write input registers
+        for (reg, slot) in self
+            .registers
+            .iter()
+            .zip(self.stack_slots_input.iter().copied())
+            .filter_map(|(r, s)| r.zip(s))
+        {
+            Self::restore_register(&mut generated_asm, self.arch, reg, slot);
+        }
+
+        if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
+            generated_asm.push_str(".att_syntax\n");
+        }
+
+        // The actual inline asm
+        for piece in self.template {
+            match piece {
+                InlineAsmTemplatePiece::String(s) => {
+                    generated_asm.push_str(s);
+                }
+                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
+                    if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
+                        generated_asm.push('%');
+                    }
+                    self.registers[*operand_idx]
+                        .unwrap()
+                        .emit(&mut generated_asm, self.arch, *modifier)
+                        .unwrap();
+                }
+            }
+        }
+        generated_asm.push('\n');
+
+        if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
+            generated_asm.push_str(".intel_syntax noprefix\n");
+        }
+
+        if !self.options.contains(InlineAsmOptions::NORETURN) {
+            // Read output registers
+            for (reg, slot) in self
+                .registers
+                .iter()
+                .zip(self.stack_slots_output.iter().copied())
+                .filter_map(|(r, s)| r.zip(s))
+            {
+                Self::save_register(&mut generated_asm, self.arch, reg, slot);
+            }
+
+            // Restore clobbered registers
+            for (reg, slot) in self
+                .registers
+                .iter()
+                .zip(self.stack_slots_clobber.iter().copied())
+                .filter_map(|(r, s)| r.zip(s))
+            {
+                Self::restore_register(&mut generated_asm, self.arch, reg, slot);
+            }
+
+            Self::epilogue(&mut generated_asm, self.arch);
+        } else {
+            Self::epilogue_noreturn(&mut generated_asm, self.arch);
+        }
+
+        if is_x86 {
+            generated_asm.push_str(".att_syntax\n");
+        }
+        writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
+        generated_asm.push_str(".text\n");
+        generated_asm.push_str("\n\n");
+
+        generated_asm
     }
 
-    if !options.contains(InlineAsmOptions::NORETURN) {
-        // Read output registers
-        for &(reg, offset, _place) in outputs {
-            save_register(&mut generated_asm, arch, reg, offset);
+    fn prologue(generated_asm: &mut String, arch: InlineAsmArch) {
+        match arch {
+            InlineAsmArch::X86 => {
+                generated_asm.push_str("    push ebp\n");
+                generated_asm.push_str("    mov ebp,[esp+8]\n");
+            }
+            InlineAsmArch::X86_64 => {
+                generated_asm.push_str("    push rbp\n");
+                generated_asm.push_str("    mov rbp,rdi\n");
+            }
+            InlineAsmArch::RiscV32 => {
+                generated_asm.push_str("    addi sp, sp, -8\n");
+                generated_asm.push_str("    sw ra, 4(sp)\n");
+                generated_asm.push_str("    sw s0, 0(sp)\n");
+                generated_asm.push_str("    mv s0, a0\n");
+            }
+            InlineAsmArch::RiscV64 => {
+                generated_asm.push_str("    addi sp, sp, -16\n");
+                generated_asm.push_str("    sd ra, 8(sp)\n");
+                generated_asm.push_str("    sd s0, 0(sp)\n");
+                generated_asm.push_str("    mv s0, a0\n");
+            }
+            _ => unimplemented!("prologue for {:?}", arch),
         }
+    }
 
-        // Restore clobbered registers
-        for &(reg, offset) in clobbered_regs.iter().rev() {
-            restore_register(&mut generated_asm, arch, reg, offset);
+    fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) {
+        match arch {
+            InlineAsmArch::X86 => {
+                generated_asm.push_str("    pop ebp\n");
+                generated_asm.push_str("    ret\n");
+            }
+            InlineAsmArch::X86_64 => {
+                generated_asm.push_str("    pop rbp\n");
+                generated_asm.push_str("    ret\n");
+            }
+            InlineAsmArch::RiscV32 => {
+                generated_asm.push_str("    lw s0, 0(sp)\n");
+                generated_asm.push_str("    lw ra, 4(sp)\n");
+                generated_asm.push_str("    addi sp, sp, 8\n");
+                generated_asm.push_str("    ret\n");
+            }
+            InlineAsmArch::RiscV64 => {
+                generated_asm.push_str("    ld s0, 0(sp)\n");
+                generated_asm.push_str("    ld ra, 8(sp)\n");
+                generated_asm.push_str("    addi sp, sp, 16\n");
+                generated_asm.push_str("    ret\n");
+            }
+            _ => unimplemented!("epilogue for {:?}", arch),
         }
+    }
 
-        generated_asm.push_str("    pop rbp\n");
-        generated_asm.push_str("    ret\n");
-    } else {
-        generated_asm.push_str("    ud2\n");
+    fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) {
+        match arch {
+            InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
+                generated_asm.push_str("    ud2\n");
+            }
+            InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
+                generated_asm.push_str("    ebreak\n");
+            }
+            _ => unimplemented!("epilogue_noreturn for {:?}", arch),
+        }
     }
 
-    generated_asm.push_str(".att_syntax\n");
-    writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
-    generated_asm.push_str(".text\n");
-    generated_asm.push_str("\n\n");
+    fn save_register(
+        generated_asm: &mut String,
+        arch: InlineAsmArch,
+        reg: InlineAsmReg,
+        offset: Size,
+    ) {
+        match arch {
+            InlineAsmArch::X86 => {
+                write!(generated_asm, "    mov [ebp+0x{:x}], ", offset.bytes()).unwrap();
+                reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap();
+                generated_asm.push('\n');
+            }
+            InlineAsmArch::X86_64 => {
+                write!(generated_asm, "    mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
+                reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
+                generated_asm.push('\n');
+            }
+            InlineAsmArch::RiscV32 => {
+                generated_asm.push_str("    sw ");
+                reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap();
+                writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
+            }
+            InlineAsmArch::RiscV64 => {
+                generated_asm.push_str("    sd ");
+                reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
+                writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
+            }
+            _ => unimplemented!("save_register for {:?}", arch),
+        }
+    }
 
-    generated_asm
+    fn restore_register(
+        generated_asm: &mut String,
+        arch: InlineAsmArch,
+        reg: InlineAsmReg,
+        offset: Size,
+    ) {
+        match arch {
+            InlineAsmArch::X86 => {
+                generated_asm.push_str("    mov ");
+                reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap();
+                writeln!(generated_asm, ", [ebp+0x{:x}]", offset.bytes()).unwrap();
+            }
+            InlineAsmArch::X86_64 => {
+                generated_asm.push_str("    mov ");
+                reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
+                writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
+            }
+            InlineAsmArch::RiscV32 => {
+                generated_asm.push_str("    lw ");
+                reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap();
+                writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
+            }
+            InlineAsmArch::RiscV64 => {
+                generated_asm.push_str("    ld ");
+                reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
+                writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
+            }
+            _ => unimplemented!("restore_register for {:?}", arch),
+        }
+    }
 }
 
 fn call_inline_asm<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     asm_name: &str,
     slot_size: Size,
-    inputs: Vec<(InlineAsmReg, Size, Value)>,
-    outputs: Vec<(InlineAsmReg, Size, CPlace<'tcx>)>,
+    inputs: Vec<(Size, Value)>,
+    outputs: Vec<(Size, CPlace<'tcx>)>,
 ) {
     let stack_slot = fx.bcx.func.create_stack_slot(StackSlotData {
         kind: StackSlotKind::ExplicitSlot,
-        offset: None,
         size: u32::try_from(slot_size.bytes()).unwrap(),
     });
     if fx.clif_comments.enabled() {
@@ -287,50 +654,16 @@ fn call_inline_asm<'tcx>(
         fx.add_comment(inline_asm_func, asm_name);
     }
 
-    for (_reg, offset, value) in inputs {
+    for (offset, value) in inputs {
         fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
     }
 
     let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
     fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
 
-    for (_reg, offset, place) in outputs {
+    for (offset, place) in outputs {
         let ty = fx.clif_type(place.layout().ty).unwrap();
         let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
         place.write_cvalue(fx, CValue::by_val(value, place.layout()));
     }
 }
-
-fn expect_reg(reg_or_class: InlineAsmRegOrRegClass) -> InlineAsmReg {
-    match reg_or_class {
-        InlineAsmRegOrRegClass::Reg(reg) => reg,
-        InlineAsmRegOrRegClass::RegClass(class) => unimplemented!("{:?}", class),
-    }
-}
-
-fn save_register(generated_asm: &mut String, arch: InlineAsmArch, reg: InlineAsmReg, offset: Size) {
-    match arch {
-        InlineAsmArch::X86_64 => {
-            write!(generated_asm, "    mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
-            reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
-            generated_asm.push('\n');
-        }
-        _ => unimplemented!("save_register for {:?}", arch),
-    }
-}
-
-fn restore_register(
-    generated_asm: &mut String,
-    arch: InlineAsmArch,
-    reg: InlineAsmReg,
-    offset: Size,
-) {
-    match arch {
-        InlineAsmArch::X86_64 => {
-            generated_asm.push_str("    mov ");
-            reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
-            writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
-        }
-        _ => unimplemented!("restore_register for {:?}", arch),
-    }
-}
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index 313b62c5770b6..f4703b22ecbcf 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -503,10 +503,10 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
 
             if intrinsic == sym::copy_nonoverlapping {
                 // FIXME emit_small_memcpy
-                fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
+                fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
             } else {
                 // FIXME emit_small_memmove
-                fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
+                fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
             }
         };
         // NOTE: the volatile variants have src and dst swapped
@@ -522,10 +522,10 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
             if intrinsic == sym::volatile_copy_nonoverlapping_memory {
                 // FIXME emit_small_memcpy
-                fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
+                fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
             } else {
                 // FIXME emit_small_memmove
-                fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
+                fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
             }
         };
         size_of_val, <T> (c ptr) {
@@ -673,7 +673,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let dst_ptr = dst.load_scalar(fx);
             // FIXME make the memset actually volatile when switching to emit_small_memset
             // FIXME use emit_small_memset
-            fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count);
+            fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
         };
         ctlz | ctlz_nonzero, <T> (v arg) {
             // FIXME trap on `ctlz_nonzero` with zero arg.
@@ -1067,7 +1067,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         kw.Try, (v f, v data, v _catch_fn) {
             // FIXME once unwinding is supported, change this to actually catch panics
             let f_sig = fx.bcx.func.import_signature(Signature {
-                call_conv: CallConv::triple_default(fx.triple()),
+                call_conv: fx.target_config.default_call_conv,
                 params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
                 returns: vec![],
             });
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index 43e68b4afa9ea..6c0631d9ecbd0 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -67,7 +67,34 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 
-            let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap();
+            // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
+            // If there is no suffix, use the index array length.
+            let n: u16 = if intrinsic == sym::simd_shuffle {
+                // Make sure this is actually an array, since typeck only checks the length-suffixed
+                // version of this intrinsic.
+                let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
+                match idx_ty.kind() {
+                    ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
+                        len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
+                            span_bug!(span, "could not evaluate shuffle index array length")
+                        }).try_into().unwrap()
+                    }
+                    _ => {
+                        fx.tcx.sess.span_err(
+                            span,
+                            &format!(
+                                "simd_shuffle index must be an array of `u32`, got `{}`",
+                                idx_ty,
+                            ),
+                        );
+                        // Prevent verifier error
+                        crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+                        return;
+                    }
+                }
+            } else {
+                intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
+            };
 
             assert_eq!(x.layout(), y.layout());
             let layout = x.layout();
@@ -378,27 +405,27 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_min, (c v) {
-            // FIXME support floats
             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
-                let lt = fx.bcx.ins().icmp(if layout.ty.is_signed() {
-                    IntCC::SignedLessThan
-                } else {
-                    IntCC::UnsignedLessThan
-                }, a, b);
+                let lt = match layout.ty.kind() {
+                    ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
+                    ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
+                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
+                    _ => unreachable!(),
+                };
                 fx.bcx.ins().select(lt, a, b)
             });
         };
 
         simd_reduce_max, (c v) {
-            // FIXME support floats
             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
-                let gt = fx.bcx.ins().icmp(if layout.ty.is_signed() {
-                    IntCC::SignedGreaterThan
-                } else {
-                    IntCC::UnsignedGreaterThan
-                }, a, b);
+                let gt = match layout.ty.kind() {
+                    ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
+                    ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
+                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),
+                    _ => unreachable!(),
+                };
                 fx.bcx.ins().select(gt, a, b)
             });
         };
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index fcdf6b50764ee..3f2884748272a 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -4,7 +4,6 @@
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
 
-extern crate snap;
 #[macro_use]
 extern crate rustc_middle;
 extern crate rustc_ast;
@@ -26,6 +25,7 @@ extern crate rustc_target;
 extern crate rustc_driver;
 
 use std::any::Any;
+use std::cell::Cell;
 
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
@@ -34,6 +34,7 @@ use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_session::config::OutputFilenames;
 use rustc_session::Session;
+use rustc_span::Symbol;
 
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::settings::{self, Configurable};
@@ -59,7 +60,6 @@ mod inline_asm;
 mod intrinsics;
 mod linkage;
 mod main_shim;
-mod metadata;
 mod num;
 mod optimize;
 mod pointer;
@@ -71,9 +71,7 @@ mod value_and_place;
 mod vtable;
 
 mod prelude {
-    pub(crate) use std::convert::{TryFrom, TryInto};
-
-    pub(crate) use rustc_span::{Span, FileNameDisplayPreference};
+    pub(crate) use rustc_span::{FileNameDisplayPreference, Span};
 
     pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
     pub(crate) use rustc_middle::bug;
@@ -125,9 +123,11 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
 struct CodegenCx<'tcx> {
     tcx: TyCtxt<'tcx>,
     global_asm: String,
+    inline_asm_index: Cell<usize>,
     cached_context: Context,
     debug_context: Option<DebugContext<'tcx>>,
     unwind_context: UnwindContext,
+    cgu_name: Symbol,
 }
 
 impl<'tcx> CodegenCx<'tcx> {
@@ -136,6 +136,7 @@ impl<'tcx> CodegenCx<'tcx> {
         backend_config: BackendConfig,
         isa: &dyn TargetIsa,
         debug_info: bool,
+        cgu_name: Symbol,
     ) -> Self {
         assert_eq!(pointer_ty(tcx), isa.pointer_type());
 
@@ -145,9 +146,11 @@ impl<'tcx> CodegenCx<'tcx> {
         CodegenCx {
             tcx,
             global_asm: String::new(),
+            inline_asm_index: Cell::new(0),
             cached_context: Context::new(),
             debug_context,
             unwind_context,
+            cgu_name,
         }
     }
 }
@@ -269,19 +272,16 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
 
     let flags = settings::Flags::new(flags_builder);
 
-    let variant = cranelift_codegen::isa::BackendVariant::MachInst;
-
     let isa_builder = match sess.opts.cg.target_cpu.as_deref() {
         Some("native") => {
-            let builder = cranelift_native::builder_with_options(variant, true).unwrap();
+            let builder = cranelift_native::builder_with_options(true).unwrap();
             builder
         }
         Some(value) => {
             let mut builder =
-                cranelift_codegen::isa::lookup_variant(target_triple.clone(), variant)
-                    .unwrap_or_else(|err| {
-                        sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
-                    });
+                cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| {
+                    sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
+                });
             if let Err(_) = builder.enable(value) {
                 sess.fatal("the specified target cpu isn't currently supported by Cranelift.");
             }
@@ -289,10 +289,9 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
         }
         None => {
             let mut builder =
-                cranelift_codegen::isa::lookup_variant(target_triple.clone(), variant)
-                    .unwrap_or_else(|err| {
-                        sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
-                    });
+                cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| {
+                    sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
+                });
             if target_triple.architecture == target_lexicon::Architecture::X86_64 {
                 // Don't use "haswell" as the default, as it implies `has_lzcnt`.
                 // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
diff --git a/compiler/rustc_codegen_cranelift/src/metadata.rs b/compiler/rustc_codegen_cranelift/src/metadata.rs
deleted file mode 100644
index 1c8fd0b01d9d9..0000000000000
--- a/compiler/rustc_codegen_cranelift/src/metadata.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-//! Writing of the rustc metadata for dylibs
-
-use object::write::{Object, StandardSegment, Symbol, SymbolSection};
-use object::{SectionKind, SymbolFlags, SymbolKind, SymbolScope};
-
-use rustc_metadata::EncodedMetadata;
-use rustc_middle::ty::TyCtxt;
-
-// Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112
-pub(crate) fn new_metadata_object(
-    tcx: TyCtxt<'_>,
-    cgu_name: &str,
-    metadata: &EncodedMetadata,
-) -> Vec<u8> {
-    use snap::write::FrameEncoder;
-    use std::io::Write;
-
-    let mut compressed = rustc_metadata::METADATA_HEADER.to_vec();
-    FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap();
-
-    let triple = crate::target_triple(tcx.sess);
-
-    let binary_format = match triple.binary_format {
-        target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf,
-        target_lexicon::BinaryFormat::Coff => object::BinaryFormat::Coff,
-        target_lexicon::BinaryFormat::Macho => object::BinaryFormat::MachO,
-        binary_format => tcx.sess.fatal(&format!("binary format {} is unsupported", binary_format)),
-    };
-    let architecture = match triple.architecture {
-        target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64,
-        target_lexicon::Architecture::Arm(_) => object::Architecture::Arm,
-        target_lexicon::Architecture::Avr => object::Architecture::Avr,
-        target_lexicon::Architecture::Hexagon => object::Architecture::Hexagon,
-        target_lexicon::Architecture::Mips32(_) => object::Architecture::Mips,
-        target_lexicon::Architecture::Mips64(_) => object::Architecture::Mips64,
-        target_lexicon::Architecture::Msp430 => object::Architecture::Msp430,
-        target_lexicon::Architecture::Powerpc => object::Architecture::PowerPc,
-        target_lexicon::Architecture::Powerpc64 => object::Architecture::PowerPc64,
-        target_lexicon::Architecture::Powerpc64le => todo!(),
-        target_lexicon::Architecture::Riscv32(_) => object::Architecture::Riscv32,
-        target_lexicon::Architecture::Riscv64(_) => object::Architecture::Riscv64,
-        target_lexicon::Architecture::S390x => object::Architecture::S390x,
-        target_lexicon::Architecture::Sparc64 => object::Architecture::Sparc64,
-        target_lexicon::Architecture::Sparcv9 => object::Architecture::Sparc64,
-        target_lexicon::Architecture::X86_32(_) => object::Architecture::I386,
-        target_lexicon::Architecture::X86_64 => object::Architecture::X86_64,
-        architecture => {
-            tcx.sess.fatal(&format!("target architecture {:?} is unsupported", architecture,))
-        }
-    };
-    let endian = match triple.endianness().unwrap() {
-        target_lexicon::Endianness::Little => object::Endianness::Little,
-        target_lexicon::Endianness::Big => object::Endianness::Big,
-    };
-
-    let mut object = Object::new(binary_format, architecture, endian);
-    object.add_file_symbol(cgu_name.as_bytes().to_vec());
-
-    let segment = object.segment_name(StandardSegment::Data).to_vec();
-    let section_id = object.add_section(segment, b".rustc".to_vec(), SectionKind::Data);
-    let offset = object.append_section_data(section_id, &compressed, 1);
-    // For MachO and probably PE this is necessary to prevent the linker from throwing away the
-    // .rustc section. For ELF this isn't necessary, but it also doesn't harm.
-    object.add_symbol(Symbol {
-        name: rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx).into_bytes(),
-        value: offset,
-        size: compressed.len() as u64,
-        kind: SymbolKind::Data,
-        scope: SymbolScope::Dynamic,
-        weak: false,
-        section: SymbolSection::Section(section_id),
-        flags: SymbolFlags::None,
-    });
-
-    object.write().unwrap()
-}
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index ec846d71960b8..4dffb89e10570 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -57,7 +57,7 @@ use std::io::Write;
 
 use cranelift_codegen::{
     entity::SecondaryMap,
-    ir::{entities::AnyEntity, function::DisplayFunctionAnnotations},
+    ir::entities::AnyEntity,
     write::{FuncWriter, PlainWriter},
 };
 
@@ -129,7 +129,6 @@ impl FuncWriter for &'_ CommentWriter {
         &mut self,
         w: &mut dyn fmt::Write,
         func: &Function,
-        reg_info: Option<&isa::RegInfo>,
     ) -> Result<bool, fmt::Error> {
         for comment in &self.global_comments {
             if !comment.is_empty() {
@@ -142,7 +141,7 @@ impl FuncWriter for &'_ CommentWriter {
             writeln!(w)?;
         }
 
-        self.super_preamble(w, func, reg_info)
+        self.super_preamble(w, func)
     }
 
     fn write_entity_definition(
@@ -165,11 +164,10 @@ impl FuncWriter for &'_ CommentWriter {
         &mut self,
         w: &mut dyn fmt::Write,
         func: &Function,
-        isa: Option<&dyn isa::TargetIsa>,
         block: Block,
         indent: usize,
     ) -> fmt::Result {
-        PlainWriter.write_block_header(w, func, isa, block, indent)
+        PlainWriter.write_block_header(w, func, block, indent)
     }
 
     fn write_instruction(
@@ -177,11 +175,10 @@ impl FuncWriter for &'_ CommentWriter {
         w: &mut dyn fmt::Write,
         func: &Function,
         aliases: &SecondaryMap<Value, Vec<Value>>,
-        isa: Option<&dyn isa::TargetIsa>,
         inst: Inst,
         indent: usize,
     ) -> fmt::Result {
-        PlainWriter.write_instruction(w, func, aliases, isa, inst, indent)?;
+        PlainWriter.write_instruction(w, func, aliases, inst, indent)?;
         if let Some(comment) = self.entity_comments.get(&inst.into()) {
             writeln!(w, "; {}", comment.replace('\n', "\n; "))?;
         }
@@ -249,7 +246,6 @@ pub(crate) fn write_clif_file<'tcx>(
                 &mut clif_comments,
                 &mut clif,
                 &context.func,
-                &DisplayFunctionAnnotations { isa: Some(isa), value_ranges: None },
             )
             .unwrap();
 
@@ -278,7 +274,6 @@ impl fmt::Debug for FunctionCx<'_, '_, '_> {
             &mut &self.clif_comments,
             &mut clif,
             &self.bcx.func,
-            &DisplayFunctionAnnotations::default(),
         )
         .unwrap();
         writeln!(f, "\n{}", clif)
diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs
index fe8d20fa39fc2..99b5366e34993 100644
--- a/compiler/rustc_codegen_cranelift/src/trap.rs
+++ b/compiler/rustc_codegen_cranelift/src/trap.rs
@@ -9,7 +9,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
             "puts",
             Linkage::Import,
             &Signature {
-                call_conv: CallConv::triple_default(fx.triple()),
+                call_conv: fx.target_config.default_call_conv,
                 params: vec![AbiParam::new(fx.pointer_type)],
                 returns: vec![AbiParam::new(types::I32)],
             },
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index 30d5340935f14..f29d13ccabddd 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -329,7 +329,6 @@ impl<'tcx> CPlace<'tcx> {
             // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
             // specify stack slot alignment.
             size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
-            offset: None,
         });
         CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
     }
@@ -472,7 +471,6 @@ impl<'tcx> CPlace<'tcx> {
                         // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
                         // specify stack slot alignment.
                         size: (src_ty.bytes() + 15) / 16 * 16,
-                        offset: None,
                     });
                     let ptr = Pointer::stack_slot(stack_slot);
                     ptr.store(fx, data, MemFlags::trusted());
@@ -512,6 +510,26 @@ impl<'tcx> CPlace<'tcx> {
         let dst_layout = self.layout();
         let to_ptr = match self.inner {
             CPlaceInner::Var(_local, var) => {
+                if let ty::Array(element, len) = dst_layout.ty.kind() {
+                    // Can only happen for vector types
+                    let len =
+                        u16::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
+                    let vector_ty = fx.clif_type(element).unwrap().by(len).unwrap();
+
+                    let data = match from.0 {
+                        CValueInner::ByRef(ptr, None) => {
+                            let mut flags = MemFlags::new();
+                            flags.set_notrap();
+                            ptr.load(fx, vector_ty, flags)
+                        }
+                        CValueInner::ByVal(_)
+                        | CValueInner::ByValPair(_, _)
+                        | CValueInner::ByRef(_, Some(_)) => bug!("array should be ByRef"),
+                    };
+
+                    fx.bcx.def_var(var, data);
+                    return;
+                }
                 let data = CValue(from.0, dst_layout).load_scalar(fx);
                 let dst_ty = fx.clif_type(self.layout().ty).unwrap();
                 transmute_value(fx, var, data, dst_ty);
@@ -583,7 +601,7 @@ impl<'tcx> CPlace<'tcx> {
                 let src_align = src_layout.align.abi.bytes() as u8;
                 let dst_align = dst_layout.align.abi.bytes() as u8;
                 fx.bcx.emit_small_memory_copy(
-                    fx.module.target_config(),
+                    fx.target_config,
                     to_addr,
                     from_addr,
                     size,
@@ -605,14 +623,39 @@ impl<'tcx> CPlace<'tcx> {
         let layout = self.layout();
 
         match self.inner {
-            CPlaceInner::Var(local, var) => {
-                if let Abi::Vector { .. } = layout.abi {
+            CPlaceInner::Var(local, var) => match layout.ty.kind() {
+                ty::Array(_, _) => {
+                    // Can only happen for vector types
                     return CPlace {
                         inner: CPlaceInner::VarLane(local, var, field.as_u32().try_into().unwrap()),
                         layout: layout.field(fx, field.as_u32().try_into().unwrap()),
                     };
                 }
-            }
+                ty::Adt(adt_def, substs) if layout.ty.is_simd() => {
+                    let f0_ty = adt_def.non_enum_variant().fields[0].ty(fx.tcx, substs);
+
+                    match f0_ty.kind() {
+                        ty::Array(_, _) => {
+                            assert_eq!(field.as_u32(), 0);
+                            return CPlace {
+                                inner: CPlaceInner::Var(local, var),
+                                layout: layout.field(fx, field.as_u32().try_into().unwrap()),
+                            };
+                        }
+                        _ => {
+                            return CPlace {
+                                inner: CPlaceInner::VarLane(
+                                    local,
+                                    var,
+                                    field.as_u32().try_into().unwrap(),
+                                ),
+                                layout: layout.field(fx, field.as_u32().try_into().unwrap()),
+                            };
+                        }
+                    }
+                }
+                _ => {}
+            },
             CPlaceInner::VarPair(local, var1, var2) => {
                 let layout = layout.field(&*fx, field.index());
 
@@ -629,7 +672,12 @@ impl<'tcx> CPlace<'tcx> {
 
         let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field);
         if field_layout.is_unsized() {
-            CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout)
+            if let ty::Foreign(_) = field_layout.ty.kind() {
+                assert!(extra.is_none());
+                CPlace::for_ptr(field_ptr, field_layout)
+            } else {
+                CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout)
+            }
         } else {
             CPlace::for_ptr(field_ptr, field_layout)
         }
diff --git a/compiler/rustc_codegen_cranelift/y.rs b/compiler/rustc_codegen_cranelift/y.rs
index 26605003c4200..98b114de91078 100755
--- a/compiler/rustc_codegen_cranelift/y.rs
+++ b/compiler/rustc_codegen_cranelift/y.rs
@@ -43,7 +43,9 @@ mod utils;
 fn usage() {
     eprintln!("Usage:");
     eprintln!("  ./y.rs prepare");
-    eprintln!("  ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]");
+    eprintln!(
+        "  ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--no-unstable-features]"
+    );
 }
 
 macro_rules! arg_error {
@@ -92,6 +94,7 @@ fn main() {
     let mut target_dir = PathBuf::from("build");
     let mut channel = "release";
     let mut sysroot_kind = SysrootKind::Clif;
+    let mut use_unstable_features = true;
     while let Some(arg) = args.next().as_deref() {
         match arg {
             "--target-dir" => {
@@ -109,6 +112,7 @@ fn main() {
                     None => arg_error!("--sysroot requires argument"),
                 }
             }
+            "--no-unstable-features" => use_unstable_features = false,
             flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
             arg => arg_error!("Unexpected argument {}", arg),
         }
@@ -141,7 +145,8 @@ fn main() {
         process::exit(1);
     }
 
-    let cg_clif_build_dir = build_backend::build_backend(channel, &host_triple);
+    let cg_clif_build_dir =
+        build_backend::build_backend(channel, &host_triple, use_unstable_features);
     build_sysroot::build_sysroot(
         channel,
         sysroot_kind,
diff --git a/compiler/rustc_lint/src/context.rs b/compiler/rustc_lint/src/context.rs
index f024b4bb74c24..69c376c6169ba 100644
--- a/compiler/rustc_lint/src/context.rs
+++ b/compiler/rustc_lint/src/context.rs
@@ -661,7 +661,7 @@ pub trait LintContext: Sized {
                 BuiltinLintDiagnostics::UnknownCrateTypes(span, note, sugg) => {
                     db.span_suggestion(span, &note, sugg, Applicability::MaybeIncorrect);
                 }
-                BuiltinLintDiagnostics::UnusedImports(message, replaces) => {
+                BuiltinLintDiagnostics::UnusedImports(message, replaces, in_test_module) => {
                     if !replaces.is_empty() {
                         db.tool_only_multipart_suggestion(
                             &message,
@@ -669,6 +669,14 @@ pub trait LintContext: Sized {
                             Applicability::MachineApplicable,
                         );
                     }
+
+                    if let Some(span) = in_test_module {
+                        let def_span = self.sess().source_map().guess_head_span(span);
+                        db.span_help(
+                            span.shrink_to_lo().to(def_span),
+                            "consider adding a `#[cfg(test)]` to the containing module",
+                        );
+                    }
                 }
                 BuiltinLintDiagnostics::RedundantImport(spans, ident) => {
                     for (span, is_imported) in spans {
diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs
index e22c9c68de6e6..97f6df51f88dd 100644
--- a/compiler/rustc_lint_defs/src/lib.rs
+++ b/compiler/rustc_lint_defs/src/lib.rs
@@ -289,7 +289,7 @@ pub enum BuiltinLintDiagnostics {
     ProcMacroDeriveResolutionFallback(Span),
     MacroExpandedMacroExportsAccessedByAbsolutePaths(Span),
     UnknownCrateTypes(Span, String, String),
-    UnusedImports(String, Vec<(Span, String)>),
+    UnusedImports(String, Vec<(Span, String)>, Option<Span>),
     RedundantImport(Vec<(Span, bool)>, Ident),
     DeprecatedMacro(Option<Symbol>, Span),
     MissingAbi(Span, Abi),
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index 5ba7efc37f8bd..4e5d21049a0d9 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -17,7 +17,7 @@ use rustc_session::utils::NativeLibKind;
 use rustc_session::{Session, StableCrateId};
 use rustc_span::hygiene::{ExpnHash, ExpnId};
 use rustc_span::source_map::{Span, Spanned};
-use rustc_span::symbol::Symbol;
+use rustc_span::symbol::{kw, Symbol};
 
 use rustc_data_structures::sync::Lrc;
 use smallvec::SmallVec;
@@ -295,6 +295,10 @@ pub fn provide(providers: &mut Providers) {
             use std::collections::vec_deque::VecDeque;
 
             let mut visible_parent_map: DefIdMap<DefId> = Default::default();
+            // This is a secondary visible_parent_map, storing the DefId of parents that re-export
+            // the child as `_`. Since we prefer parents that don't do this, merge this map at the
+            // end, only if we're missing any keys from the former.
+            let mut fallback_map: DefIdMap<DefId> = Default::default();
 
             // Issue 46112: We want the map to prefer the shortest
             // paths when reporting the path to an item. Therefore we
@@ -317,12 +321,17 @@ pub fn provide(providers: &mut Providers) {
                 bfs_queue.push_back(DefId { krate: cnum, index: CRATE_DEF_INDEX });
             }
 
-            let mut add_child = |bfs_queue: &mut VecDeque<_>, child: &Export, parent: DefId| {
-                if !child.vis.is_public() {
+            let mut add_child = |bfs_queue: &mut VecDeque<_>, export: &Export, parent: DefId| {
+                if !export.vis.is_public() {
                     return;
                 }
 
-                if let Some(child) = child.res.opt_def_id() {
+                if let Some(child) = export.res.opt_def_id() {
+                    if export.ident.name == kw::Underscore {
+                        fallback_map.insert(child, parent);
+                        return;
+                    }
+
                     match visible_parent_map.entry(child) {
                         Entry::Occupied(mut entry) => {
                             // If `child` is defined in crate `cnum`, ensure
@@ -345,6 +354,12 @@ pub fn provide(providers: &mut Providers) {
                 }
             }
 
+            // Fill in any missing entries with the (less preferable) path ending in `::_`.
+            // We still use this path in a diagnostic that suggests importing `::*`.
+            for (child, parent) in fallback_map {
+                visible_parent_map.entry(child).or_insert(parent);
+            }
+
             visible_parent_map
         },
 
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 3faedf242860e..47a9234419c2d 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -319,6 +319,9 @@ pub trait PrettyPrinter<'tcx>:
     ///
     /// `callers` is a chain of visible_parent's leading to `def_id`,
     /// to support cycle detection during recursion.
+    ///
+    /// This method returns false if we can't print the visible path, so
+    /// `print_def_path` can fall back on the item's real definition path.
     fn try_print_visible_def_path_recur(
         mut self,
         def_id: DefId,
@@ -405,19 +408,7 @@ pub trait PrettyPrinter<'tcx>:
             Some(parent) => parent,
             None => return Ok((self, false)),
         };
-        if callers.contains(&visible_parent) {
-            return Ok((self, false));
-        }
-        callers.push(visible_parent);
-        // HACK(eddyb) this bypasses `path_append`'s prefix printing to avoid
-        // knowing ahead of time whether the entire path will succeed or not.
-        // To support printers that do not implement `PrettyPrinter`, a `Vec` or
-        // linked list on the stack would need to be built, before any printing.
-        match self.try_print_visible_def_path_recur(visible_parent, callers)? {
-            (cx, false) => return Ok((cx, false)),
-            (cx, true) => self = cx,
-        }
-        callers.pop();
+
         let actual_parent = self.tcx().parent(def_id);
         debug!(
             "try_print_visible_def_path: visible_parent={:?} actual_parent={:?}",
@@ -463,14 +454,21 @@ pub trait PrettyPrinter<'tcx>:
             // `visible_parent_map`), looking for the specific child we currently have and then
             // have access to the re-exported name.
             DefPathData::TypeNs(ref mut name) if Some(visible_parent) != actual_parent => {
+                // Item might be re-exported several times, but filter for the one
+                // that's public and whose identifier isn't `_`.
                 let reexport = self
                     .tcx()
                     .item_children(visible_parent)
                     .iter()
-                    .find(|child| child.res.opt_def_id() == Some(def_id))
+                    .filter(|child| child.res.opt_def_id() == Some(def_id))
+                    .find(|child| child.vis.is_public() && child.ident.name != kw::Underscore)
                     .map(|child| child.ident.name);
-                if let Some(reexport) = reexport {
-                    *name = reexport;
+
+                if let Some(new_name) = reexport {
+                    *name = new_name;
+                } else {
+                    // There is no name that is public and isn't `_`, so bail.
+                    return Ok((self, false));
                 }
             }
             // Re-exported `extern crate` (#43189).
@@ -481,6 +479,20 @@ pub trait PrettyPrinter<'tcx>:
         }
         debug!("try_print_visible_def_path: data={:?}", data);
 
+        if callers.contains(&visible_parent) {
+            return Ok((self, false));
+        }
+        callers.push(visible_parent);
+        // HACK(eddyb) this bypasses `path_append`'s prefix printing to avoid
+        // knowing ahead of time whether the entire path will succeed or not.
+        // To support printers that do not implement `PrettyPrinter`, a `Vec` or
+        // linked list on the stack would need to be built, before any printing.
+        match self.try_print_visible_def_path_recur(visible_parent, callers)? {
+            (cx, false) => return Ok((cx, false)),
+            (cx, true) => self = cx,
+        }
+        callers.pop();
+
         Ok((self.path_append(Ok, &DisambiguatedDefPathData { data, disambiguator: 0 })?, true))
     }
 
diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs
index d45c064d5e37e..74edc3a2d5e6f 100644
--- a/compiler/rustc_resolve/src/build_reduced_graph.rs
+++ b/compiler/rustc_resolve/src/build_reduced_graph.rs
@@ -108,7 +108,7 @@ impl<'a> Resolver<'a> {
     /// Reachable macros with block module parents exist due to `#[macro_export] macro_rules!`,
     /// but they cannot use def-site hygiene, so the assumption holds
     /// (<https://github.com/rust-lang/rust/pull/77984#issuecomment-712445508>).
-    fn get_nearest_non_block_module(&mut self, mut def_id: DefId) -> Module<'a> {
+    crate fn get_nearest_non_block_module(&mut self, mut def_id: DefId) -> Module<'a> {
         loop {
             match self.get_module(def_id) {
                 Some(module) => return module,
diff --git a/compiler/rustc_resolve/src/check_unused.rs b/compiler/rustc_resolve/src/check_unused.rs
index 63699128e9e16..601f2d96ff5eb 100644
--- a/compiler/rustc_resolve/src/check_unused.rs
+++ b/compiler/rustc_resolve/src/check_unused.rs
@@ -24,6 +24,7 @@
 //    in the last step
 
 use crate::imports::ImportKind;
+use crate::module_to_string;
 use crate::Resolver;
 
 use rustc_ast as ast;
@@ -314,12 +315,29 @@ impl Resolver<'_> {
                 "remove the unused import"
             };
 
+            let parent_module = visitor.r.get_nearest_non_block_module(
+                visitor.r.local_def_id(unused.use_tree_id).to_def_id(),
+            );
+            let test_module_span = match module_to_string(parent_module) {
+                Some(module)
+                    if module == "test"
+                        || module == "tests"
+                        || module.starts_with("test_")
+                        || module.starts_with("tests_")
+                        || module.ends_with("_test")
+                        || module.ends_with("_tests") =>
+                {
+                    Some(parent_module.span)
+                }
+                _ => None,
+            };
+
             visitor.r.lint_buffer.buffer_lint_with_diagnostic(
                 UNUSED_IMPORTS,
                 unused.use_tree_id,
                 ms,
                 &msg,
-                BuiltinLintDiagnostics::UnusedImports(fix_msg.into(), fixes),
+                BuiltinLintDiagnostics::UnusedImports(fix_msg.into(), fixes, test_module_span),
             );
         }
     }
diff --git a/compiler/rustc_typeck/src/check/method/suggest.rs b/compiler/rustc_typeck/src/check/method/suggest.rs
index 1dcc20c29a32c..d5a4de86d4d1e 100644
--- a/compiler/rustc_typeck/src/check/method/suggest.rs
+++ b/compiler/rustc_typeck/src/check/method/suggest.rs
@@ -12,7 +12,7 @@ use rustc_hir::{ExprKind, Node, QPath};
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_middle::ty::fast_reject::{simplify_type, SimplifyParams, StripReferences};
 use rustc_middle::ty::print::with_crate_prefix;
-use rustc_middle::ty::{self, ToPredicate, Ty, TyCtxt, TypeFoldable};
+use rustc_middle::ty::{self, DefIdTree, ToPredicate, Ty, TyCtxt, TypeFoldable};
 use rustc_span::lev_distance;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::{source_map, FileName, MultiSpan, Span, Symbol};
@@ -1310,25 +1310,66 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         mut msg: String,
         candidates: Vec<DefId>,
     ) {
+        let parent_map = self.tcx.visible_parent_map(());
+
+        // Separate out candidates that must be imported with a glob, because they are named `_`
+        // and cannot be referred with their identifier.
+        let (candidates, globs): (Vec<_>, Vec<_>) = candidates.into_iter().partition(|trait_did| {
+            if let Some(parent_did) = parent_map.get(trait_did) {
+                // If the item is re-exported as `_`, we should suggest a glob-import instead.
+                if Some(*parent_did) != self.tcx.parent(*trait_did)
+                    && self
+                        .tcx
+                        .item_children(*parent_did)
+                        .iter()
+                        .filter(|child| child.res.opt_def_id() == Some(*trait_did))
+                        .all(|child| child.ident.name == kw::Underscore)
+                {
+                    return false;
+                }
+            }
+
+            true
+        });
+
         let module_did = self.tcx.parent_module(self.body_id);
         let (span, found_use) = find_use_placement(self.tcx, module_did);
         if let Some(span) = span {
-            let path_strings = candidates.iter().map(|did| {
+            let path_strings = candidates.iter().map(|trait_did| {
                 // Produce an additional newline to separate the new use statement
                 // from the directly following item.
                 let additional_newline = if found_use { "" } else { "\n" };
                 format!(
                     "use {};\n{}",
-                    with_crate_prefix(|| self.tcx.def_path_str(*did)),
+                    with_crate_prefix(|| self.tcx.def_path_str(*trait_did)),
                     additional_newline
                 )
             });
 
-            err.span_suggestions(span, &msg, path_strings, Applicability::MaybeIncorrect);
+            let glob_path_strings = globs.iter().map(|trait_did| {
+                let parent_did = parent_map.get(trait_did).unwrap();
+
+                // Produce an additional newline to separate the new use statement
+                // from the directly following item.
+                let additional_newline = if found_use { "" } else { "\n" };
+                format!(
+                    "use {}::*; // trait {}\n{}",
+                    with_crate_prefix(|| self.tcx.def_path_str(*parent_did)),
+                    self.tcx.item_name(*trait_did),
+                    additional_newline
+                )
+            });
+
+            err.span_suggestions(
+                span,
+                &msg,
+                path_strings.chain(glob_path_strings),
+                Applicability::MaybeIncorrect,
+            );
         } else {
-            let limit = if candidates.len() == 5 { 5 } else { 4 };
+            let limit = if candidates.len() + globs.len() == 5 { 5 } else { 4 };
             for (i, trait_did) in candidates.iter().take(limit).enumerate() {
-                if candidates.len() > 1 {
+                if candidates.len() + globs.len() > 1 {
                     msg.push_str(&format!(
                         "\ncandidate #{}: `use {};`",
                         i + 1,
@@ -1341,8 +1382,28 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     ));
                 }
             }
+            for (i, trait_did) in
+                globs.iter().take(limit.saturating_sub(candidates.len())).enumerate()
+            {
+                let parent_did = parent_map.get(trait_did).unwrap();
+
+                if candidates.len() + globs.len() > 1 {
+                    msg.push_str(&format!(
+                        "\ncandidate #{}: `use {}::*; // trait {}`",
+                        candidates.len() + i + 1,
+                        with_crate_prefix(|| self.tcx.def_path_str(*parent_did)),
+                        self.tcx.item_name(*trait_did),
+                    ));
+                } else {
+                    msg.push_str(&format!(
+                        "\n`use {}::*; // trait {}`",
+                        with_crate_prefix(|| self.tcx.def_path_str(*parent_did)),
+                        self.tcx.item_name(*trait_did),
+                    ));
+                }
+            }
             if candidates.len() > limit {
-                msg.push_str(&format!("\nand {} others", candidates.len() - limit));
+                msg.push_str(&format!("\nand {} others", candidates.len() + globs.len() - limit));
             }
             err.note(&msg);
         }
diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs
index 332be06dd1617..a93327a0132ed 100644
--- a/library/core/src/ptr/const_ptr.rs
+++ b/library/core/src/ptr/const_ptr.rs
@@ -163,8 +163,9 @@ impl<T: ?Sized> *const T {
     /// }
     /// ```
     #[stable(feature = "ptr_as_ref", since = "1.9.0")]
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
     #[inline]
-    pub unsafe fn as_ref<'a>(self) -> Option<&'a T> {
+    pub const unsafe fn as_ref<'a>(self) -> Option<&'a T> {
         // SAFETY: the caller must guarantee that `self` is valid
         // for a reference if it isn't null.
         if self.is_null() { None } else { unsafe { Some(&*self) } }
@@ -211,7 +212,8 @@ impl<T: ?Sized> *const T {
     /// ```
     #[inline]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_ref<'a>(self) -> Option<&'a MaybeUninit<T>>
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_ref<'a>(self) -> Option<&'a MaybeUninit<T>>
     where
         T: Sized,
     {
@@ -1068,7 +1070,8 @@ impl<T> *const [T] {
     /// [allocated object]: crate::ptr#allocated-object
     #[inline]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_slice<'a>(self) -> Option<&'a [MaybeUninit<T>]> {
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_slice<'a>(self) -> Option<&'a [MaybeUninit<T>]> {
         if self.is_null() {
             None
         } else {
diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs
index f3655edb3d080..5fd3b2ebc6098 100644
--- a/library/core/src/ptr/mut_ptr.rs
+++ b/library/core/src/ptr/mut_ptr.rs
@@ -166,8 +166,9 @@ impl<T: ?Sized> *mut T {
     /// }
     /// ```
     #[stable(feature = "ptr_as_ref", since = "1.9.0")]
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
     #[inline]
-    pub unsafe fn as_ref<'a>(self) -> Option<&'a T> {
+    pub const unsafe fn as_ref<'a>(self) -> Option<&'a T> {
         // SAFETY: the caller must guarantee that `self` is valid for a
         // reference if it isn't null.
         if self.is_null() { None } else { unsafe { Some(&*self) } }
@@ -217,7 +218,8 @@ impl<T: ?Sized> *mut T {
     /// ```
     #[inline]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_ref<'a>(self) -> Option<&'a MaybeUninit<T>>
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_ref<'a>(self) -> Option<&'a MaybeUninit<T>>
     where
         T: Sized,
     {
@@ -411,8 +413,9 @@ impl<T: ?Sized> *mut T {
     /// println!("{:?}", s); // It'll print: "[4, 2, 3]".
     /// ```
     #[stable(feature = "ptr_as_ref", since = "1.9.0")]
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
     #[inline]
-    pub unsafe fn as_mut<'a>(self) -> Option<&'a mut T> {
+    pub const unsafe fn as_mut<'a>(self) -> Option<&'a mut T> {
         // SAFETY: the caller must guarantee that `self` is be valid for
         // a mutable reference if it isn't null.
         if self.is_null() { None } else { unsafe { Some(&mut *self) } }
@@ -446,7 +449,8 @@ impl<T: ?Sized> *mut T {
     /// [the module documentation]: crate::ptr#safety
     #[inline]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_mut<'a>(self) -> Option<&'a mut MaybeUninit<T>>
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_mut<'a>(self) -> Option<&'a mut MaybeUninit<T>>
     where
         T: Sized,
     {
@@ -1335,7 +1339,8 @@ impl<T> *mut [T] {
     /// [allocated object]: crate::ptr#allocated-object
     #[inline]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_slice<'a>(self) -> Option<&'a [MaybeUninit<T>]> {
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_slice<'a>(self) -> Option<&'a [MaybeUninit<T>]> {
         if self.is_null() {
             None
         } else {
@@ -1386,7 +1391,8 @@ impl<T> *mut [T] {
     /// [allocated object]: crate::ptr#allocated-object
     #[inline]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_slice_mut<'a>(self) -> Option<&'a mut [MaybeUninit<T>]> {
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_slice_mut<'a>(self) -> Option<&'a mut [MaybeUninit<T>]> {
         if self.is_null() {
             None
         } else {
diff --git a/library/core/src/ptr/non_null.rs b/library/core/src/ptr/non_null.rs
index 4f4e7eca28180..3a7e99faccf23 100644
--- a/library/core/src/ptr/non_null.rs
+++ b/library/core/src/ptr/non_null.rs
@@ -122,7 +122,8 @@ impl<T: Sized> NonNull<T> {
     #[inline]
     #[must_use]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_ref<'a>(&self) -> &'a MaybeUninit<T> {
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_ref<'a>(&self) -> &'a MaybeUninit<T> {
         // SAFETY: the caller must guarantee that `self` meets all the
         // requirements for a reference.
         unsafe { &*self.cast().as_ptr() }
@@ -155,7 +156,8 @@ impl<T: Sized> NonNull<T> {
     #[inline]
     #[must_use]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_mut<'a>(&mut self) -> &'a mut MaybeUninit<T> {
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_mut<'a>(&mut self) -> &'a mut MaybeUninit<T> {
         // SAFETY: the caller must guarantee that `self` meets all the
         // requirements for a reference.
         unsafe { &mut *self.cast().as_ptr() }
@@ -316,9 +318,10 @@ impl<T: ?Sized> NonNull<T> {
     ///
     /// [the module documentation]: crate::ptr#safety
     #[stable(feature = "nonnull", since = "1.25.0")]
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
     #[must_use]
     #[inline]
-    pub unsafe fn as_ref<'a>(&self) -> &'a T {
+    pub const unsafe fn as_ref<'a>(&self) -> &'a T {
         // SAFETY: the caller must guarantee that `self` meets all the
         // requirements for a reference.
         unsafe { &*self.as_ptr() }
@@ -366,9 +369,10 @@ impl<T: ?Sized> NonNull<T> {
     ///
     /// [the module documentation]: crate::ptr#safety
     #[stable(feature = "nonnull", since = "1.25.0")]
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
     #[must_use]
     #[inline]
-    pub unsafe fn as_mut<'a>(&mut self) -> &'a mut T {
+    pub const unsafe fn as_mut<'a>(&mut self) -> &'a mut T {
         // SAFETY: the caller must guarantee that `self` meets all the
         // requirements for a mutable reference.
         unsafe { &mut *self.as_ptr() }
@@ -534,7 +538,8 @@ impl<T> NonNull<[T]> {
     #[inline]
     #[must_use]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_slice<'a>(&self) -> &'a [MaybeUninit<T>] {
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_slice<'a>(&self) -> &'a [MaybeUninit<T>] {
         // SAFETY: the caller must uphold the safety contract for `as_uninit_slice`.
         unsafe { slice::from_raw_parts(self.cast().as_ptr(), self.len()) }
     }
@@ -596,7 +601,8 @@ impl<T> NonNull<[T]> {
     #[inline]
     #[must_use]
     #[unstable(feature = "ptr_as_uninit", issue = "75402")]
-    pub unsafe fn as_uninit_slice_mut<'a>(&self) -> &'a mut [MaybeUninit<T>] {
+    #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")]
+    pub const unsafe fn as_uninit_slice_mut<'a>(&self) -> &'a mut [MaybeUninit<T>] {
         // SAFETY: the caller must uphold the safety contract for `as_uninit_slice_mut`.
         unsafe { slice::from_raw_parts_mut(self.cast().as_ptr(), self.len()) }
     }
diff --git a/library/std/src/collections/hash/map.rs b/library/std/src/collections/hash/map.rs
index 35f17aa781f4e..5ef23871e8b5f 100644
--- a/library/std/src/collections/hash/map.rs
+++ b/library/std/src/collections/hash/map.rs
@@ -2462,17 +2462,16 @@ impl<'a, K, V> Entry<'a, K, V> {
     /// # Examples
     ///
     /// ```
-    /// #![feature(entry_insert)]
     /// use std::collections::HashMap;
     ///
     /// let mut map: HashMap<&str, String> = HashMap::new();
-    /// let entry = map.entry("poneyland").insert("hoho".to_string());
+    /// let entry = map.entry("poneyland").insert_entry("hoho".to_string());
     ///
     /// assert_eq!(entry.key(), &"poneyland");
     /// ```
     #[inline]
-    #[unstable(feature = "entry_insert", issue = "65225")]
-    pub fn insert(self, value: V) -> OccupiedEntry<'a, K, V> {
+    #[stable(feature = "entry_insert", since = "1.59.0")]
+    pub fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V> {
         match self {
             Occupied(mut entry) => {
                 entry.insert(value);
@@ -2811,12 +2810,13 @@ impl<'a, K: 'a, V: 'a> VacantEntry<'a, K, V> {
     /// let mut map: HashMap<&str, u32> = HashMap::new();
     ///
     /// if let Entry::Vacant(o) = map.entry("poneyland") {
-    ///     o.insert(37);
+    ///     o.insert_entry(37);
     /// }
     /// assert_eq!(map["poneyland"], 37);
     /// ```
     #[inline]
-    fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V> {
+    #[stable(feature = "entry_insert", since = "1.59.0")]
+    pub fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V> {
         let base = self.base.insert_entry(value);
         OccupiedEntry { base }
     }
diff --git a/library/std/src/thread/mod.rs b/library/std/src/thread/mod.rs
index 64f6c7fa022fc..bcf2ec06022d9 100644
--- a/library/std/src/thread/mod.rs
+++ b/library/std/src/thread/mod.rs
@@ -1273,7 +1273,7 @@ impl<T> JoinInner<T> {
 /// An owned permission to join on a thread (block on its termination).
 ///
 /// A `JoinHandle` *detaches* the associated thread when it is dropped, which
-/// means that there is no longer any handle to thread and no way to `join`
+/// means that there is no longer any handle to the thread and no way to `join`
 /// on it.
 ///
 /// Due to platform restrictions, it is not possible to [`Clone`] this
diff --git a/src/librustdoc/html/static/js/search.js b/src/librustdoc/html/static/js/search.js
index d419e384a594a..971bd3930cffd 100644
--- a/src/librustdoc/html/static/js/search.js
+++ b/src/librustdoc/html/static/js/search.js
@@ -152,16 +152,26 @@ window.initSearch = function(rawSearchIndex) {
         removeEmptyStringsFromArray(split);
 
         function transformResults(results) {
+            var duplicates = {};
             var out = [];
+
             for (var i = 0, len = results.length; i < len; ++i) {
-                if (results[i].id > -1) {
-                    var obj = searchIndex[results[i].id];
-                    obj.lev = results[i].lev;
+                var result = results[i];
+
+                if (result.id > -1) {
+                    var obj = searchIndex[result.id];
+                    obj.lev = result.lev;
                     var res = buildHrefAndPath(obj);
                     obj.displayPath = pathSplitter(res[0]);
                     obj.fullPath = obj.displayPath + obj.name;
                     // To be sure than it some items aren't considered as duplicate.
                     obj.fullPath += "|" + obj.ty;
+
+                    if (duplicates[obj.fullPath]) {
+                        continue;
+                    }
+                    duplicates[obj.fullPath] = true;
+
                     obj.href = res[1];
                     out.push(obj);
                     if (out.length >= MAX_RESULTS) {
@@ -971,19 +981,11 @@ window.initSearch = function(rawSearchIndex) {
         }
 
         var output = document.createElement("div");
-        var duplicates = {};
         var length = 0;
         if (array.length > 0) {
             output.className = "search-results " + extraClass;
 
             array.forEach(function(item) {
-                if (item.is_alias !== true) {
-                    if (duplicates[item.fullPath]) {
-                        return;
-                    }
-                    duplicates[item.fullPath] = true;
-                }
-
                 var name = item.name;
                 var type = itemTypes[item.ty];
 
diff --git a/src/test/ui/imports/auxiliary/overlapping_pub_trait_source.rs b/src/test/ui/imports/auxiliary/overlapping_pub_trait_source.rs
new file mode 100644
index 0000000000000..2a25d60acff7b
--- /dev/null
+++ b/src/test/ui/imports/auxiliary/overlapping_pub_trait_source.rs
@@ -0,0 +1,13 @@
+/* This crate declares an item as both `prelude::*` and `m::Tr`.
+ * The compiler should always suggest `m::Tr`. */
+
+pub struct S;
+
+pub mod prelude {
+    pub use crate::m::Tr as _;
+}
+
+pub mod m {
+    pub trait Tr { fn method(&self); }
+    impl Tr for crate::S { fn method(&self) {} }
+}
diff --git a/src/test/ui/imports/auxiliary/unnamed_pub_trait_source.rs b/src/test/ui/imports/auxiliary/unnamed_pub_trait_source.rs
new file mode 100644
index 0000000000000..d73c9a795b673
--- /dev/null
+++ b/src/test/ui/imports/auxiliary/unnamed_pub_trait_source.rs
@@ -0,0 +1,13 @@
+/* This crate declares an item that is unnamed.
+ * Its only public path is through `prelude::*`. */
+
+pub struct S;
+
+mod m {
+    pub trait Tr { fn method(&self); }
+    impl Tr for crate::S { fn method(&self) {} }
+}
+
+pub mod prelude {
+    pub use crate::m::Tr as _;
+}
diff --git a/src/test/ui/imports/overlapping_pub_trait.rs b/src/test/ui/imports/overlapping_pub_trait.rs
new file mode 100644
index 0000000000000..f5f5d4ed3804f
--- /dev/null
+++ b/src/test/ui/imports/overlapping_pub_trait.rs
@@ -0,0 +1,15 @@
+// aux-build:overlapping_pub_trait_source.rs
+
+/*
+ * This crate declares two public paths, `m::Tr` and `prelude::_`. Make sure we prefer the former.
+ */
+extern crate overlapping_pub_trait_source;
+
+fn main() {
+    //~^ HELP the following trait is implemented but not in scope; perhaps add a `use` for it:
+    //~| SUGGESTION overlapping_pub_trait_source::m::Tr
+    use overlapping_pub_trait_source::S;
+    S.method();
+    //~^ ERROR no method named `method` found for struct `S` in the current scope [E0599]
+    //~| HELP items from traits can only be used if the trait is in scope
+}
diff --git a/src/test/ui/imports/overlapping_pub_trait.stderr b/src/test/ui/imports/overlapping_pub_trait.stderr
new file mode 100644
index 0000000000000..d0c845a5e523f
--- /dev/null
+++ b/src/test/ui/imports/overlapping_pub_trait.stderr
@@ -0,0 +1,20 @@
+error[E0599]: no method named `method` found for struct `S` in the current scope
+  --> $DIR/overlapping_pub_trait.rs:12:7
+   |
+LL |     S.method();
+   |       ^^^^^^ method not found in `S`
+   |
+  ::: $DIR/auxiliary/overlapping_pub_trait_source.rs:11:23
+   |
+LL |     pub trait Tr { fn method(&self); }
+   |                       ------ the method is available for `S` here
+   |
+   = help: items from traits can only be used if the trait is in scope
+help: the following trait is implemented but not in scope; perhaps add a `use` for it:
+   |
+LL | use overlapping_pub_trait_source::m::Tr;
+   |
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0599`.
diff --git a/src/test/ui/imports/unnamed_pub_trait.rs b/src/test/ui/imports/unnamed_pub_trait.rs
new file mode 100644
index 0000000000000..b06b1e1d07dce
--- /dev/null
+++ b/src/test/ui/imports/unnamed_pub_trait.rs
@@ -0,0 +1,16 @@
+// aux-build:unnamed_pub_trait_source.rs
+
+/*
+ * This crate declares an unnameable public path for our item. Make sure we don't suggest
+ * importing it by name, and instead we suggest importing it by glob.
+ */
+extern crate unnamed_pub_trait_source;
+
+fn main() {
+    //~^ HELP the following trait is implemented but not in scope; perhaps add a `use` for it:
+    //~| SUGGESTION unnamed_pub_trait_source::prelude::*; // trait Tr
+    use unnamed_pub_trait_source::S;
+    S.method();
+    //~^ ERROR no method named `method` found for struct `S` in the current scope [E0599]
+    //~| HELP items from traits can only be used if the trait is in scope
+}
diff --git a/src/test/ui/imports/unnamed_pub_trait.stderr b/src/test/ui/imports/unnamed_pub_trait.stderr
new file mode 100644
index 0000000000000..319dfd7e1b28b
--- /dev/null
+++ b/src/test/ui/imports/unnamed_pub_trait.stderr
@@ -0,0 +1,20 @@
+error[E0599]: no method named `method` found for struct `S` in the current scope
+  --> $DIR/unnamed_pub_trait.rs:13:7
+   |
+LL |     S.method();
+   |       ^^^^^^ method not found in `S`
+   |
+  ::: $DIR/auxiliary/unnamed_pub_trait_source.rs:7:23
+   |
+LL |     pub trait Tr { fn method(&self); }
+   |                       ------ the method is available for `S` here
+   |
+   = help: items from traits can only be used if the trait is in scope
+help: the following trait is implemented but not in scope; perhaps add a `use` for it:
+   |
+LL | use unnamed_pub_trait_source::prelude::*; // trait Tr
+   |
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0599`.
diff --git a/src/test/ui/imports/unused-imports-in-test-module.rs b/src/test/ui/imports/unused-imports-in-test-module.rs
new file mode 100644
index 0000000000000..b003b99b6cff0
--- /dev/null
+++ b/src/test/ui/imports/unused-imports-in-test-module.rs
@@ -0,0 +1,64 @@
+#![deny(unused_imports)]
+
+use std::io::BufRead; //~ ERROR unused import: `std::io::BufRead`
+
+fn a() {}
+fn b() {}
+
+mod test {
+    use super::a;  //~ ERROR unused import: `super::a`
+
+    fn foo() {
+        use crate::b;  //~ ERROR unused import: `crate::b`
+    }
+}
+
+mod tests {
+    use super::a;  //~ ERROR unused import: `super::a`
+
+    fn foo() {
+        use crate::b;  //~ ERROR unused import: `crate::b`
+    }
+}
+
+mod test_a {
+    use super::a;  //~ ERROR unused import: `super::a`
+
+    fn foo() {
+        use crate::b;  //~ ERROR unused import: `crate::b`
+    }
+}
+
+mod a_test {
+    use super::a;  //~ ERROR unused import: `super::a`
+
+    fn foo() {
+        use crate::b;  //~ ERROR unused import: `crate::b`
+    }
+}
+
+mod tests_a {
+    use super::a;  //~ ERROR unused import: `super::a`
+
+    fn foo() {
+        use crate::b;  //~ ERROR unused import: `crate::b`
+    }
+}
+
+mod a_tests {
+    use super::a;  //~ ERROR unused import: `super::a`
+
+    fn foo() {
+        use crate::b;  //~ ERROR unused import: `crate::b`
+    }
+}
+
+mod fastest_search {
+    use super::a;  //~ ERROR unused import: `super::a`
+
+    fn foo() {
+        use crate::b;  //~ ERROR unused import: `crate::b`
+    }
+}
+
+fn main() {}
diff --git a/src/test/ui/imports/unused-imports-in-test-module.stderr b/src/test/ui/imports/unused-imports-in-test-module.stderr
new file mode 100644
index 0000000000000..2efea5b3609e1
--- /dev/null
+++ b/src/test/ui/imports/unused-imports-in-test-module.stderr
@@ -0,0 +1,170 @@
+error: unused import: `std::io::BufRead`
+  --> $DIR/unused-imports-in-test-module.rs:3:5
+   |
+LL | use std::io::BufRead;
+   |     ^^^^^^^^^^^^^^^^
+   |
+note: the lint level is defined here
+  --> $DIR/unused-imports-in-test-module.rs:1:9
+   |
+LL | #![deny(unused_imports)]
+   |         ^^^^^^^^^^^^^^
+
+error: unused import: `super::a`
+  --> $DIR/unused-imports-in-test-module.rs:9:9
+   |
+LL |     use super::a;
+   |         ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:8:1
+   |
+LL | mod test {
+   | ^^^^^^^^
+
+error: unused import: `crate::b`
+  --> $DIR/unused-imports-in-test-module.rs:12:13
+   |
+LL |         use crate::b;
+   |             ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:8:1
+   |
+LL | mod test {
+   | ^^^^^^^^
+
+error: unused import: `super::a`
+  --> $DIR/unused-imports-in-test-module.rs:17:9
+   |
+LL |     use super::a;
+   |         ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:16:1
+   |
+LL | mod tests {
+   | ^^^^^^^^^
+
+error: unused import: `crate::b`
+  --> $DIR/unused-imports-in-test-module.rs:20:13
+   |
+LL |         use crate::b;
+   |             ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:16:1
+   |
+LL | mod tests {
+   | ^^^^^^^^^
+
+error: unused import: `super::a`
+  --> $DIR/unused-imports-in-test-module.rs:25:9
+   |
+LL |     use super::a;
+   |         ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:24:1
+   |
+LL | mod test_a {
+   | ^^^^^^^^^^
+
+error: unused import: `crate::b`
+  --> $DIR/unused-imports-in-test-module.rs:28:13
+   |
+LL |         use crate::b;
+   |             ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:24:1
+   |
+LL | mod test_a {
+   | ^^^^^^^^^^
+
+error: unused import: `super::a`
+  --> $DIR/unused-imports-in-test-module.rs:33:9
+   |
+LL |     use super::a;
+   |         ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:32:1
+   |
+LL | mod a_test {
+   | ^^^^^^^^^^
+
+error: unused import: `crate::b`
+  --> $DIR/unused-imports-in-test-module.rs:36:13
+   |
+LL |         use crate::b;
+   |             ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:32:1
+   |
+LL | mod a_test {
+   | ^^^^^^^^^^
+
+error: unused import: `super::a`
+  --> $DIR/unused-imports-in-test-module.rs:41:9
+   |
+LL |     use super::a;
+   |         ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:40:1
+   |
+LL | mod tests_a {
+   | ^^^^^^^^^^^
+
+error: unused import: `crate::b`
+  --> $DIR/unused-imports-in-test-module.rs:44:13
+   |
+LL |         use crate::b;
+   |             ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:40:1
+   |
+LL | mod tests_a {
+   | ^^^^^^^^^^^
+
+error: unused import: `super::a`
+  --> $DIR/unused-imports-in-test-module.rs:49:9
+   |
+LL |     use super::a;
+   |         ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:48:1
+   |
+LL | mod a_tests {
+   | ^^^^^^^^^^^
+
+error: unused import: `crate::b`
+  --> $DIR/unused-imports-in-test-module.rs:52:13
+   |
+LL |         use crate::b;
+   |             ^^^^^^^^
+   |
+help: consider adding a `#[cfg(test)]` to the containing module
+  --> $DIR/unused-imports-in-test-module.rs:48:1
+   |
+LL | mod a_tests {
+   | ^^^^^^^^^^^
+
+error: unused import: `super::a`
+  --> $DIR/unused-imports-in-test-module.rs:57:9
+   |
+LL |     use super::a;
+   |         ^^^^^^^^
+
+error: unused import: `crate::b`
+  --> $DIR/unused-imports-in-test-module.rs:60:13
+   |
+LL |         use crate::b;
+   |             ^^^^^^^^
+
+error: aborting due to 15 previous errors
+