Tweak assert_unsafe_precondition helpers so they optimize well in MIR

compiler/rustc_mir_transform/src/const_prop.rs

@@ -57,7 +57,14 @@ macro_rules! throw_machine_stop_str {
     }};
 }
 
-pub struct ConstProp;
+pub struct ConstProp {
+    only_bools: bool,
+}
+impl ConstProp {
+    pub fn new(only_bools: bool) -> Self {
+        ConstProp { only_bools }
+    }
+}
 
 impl<'tcx> MirPass<'tcx> for ConstProp {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
@@ -149,7 +156,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
         // constants, instead of just checking for const-folding succeeding.
         // That would require a uniform one-def no-mutation analysis
         // and RPO (or recursing when needing the value of a local).
-        let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx);
+        let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx, self.only_bools);
         optimization_finder.visit_body(body);
 
         trace!("ConstProp done for {:?}", def_id);
@@ -373,12 +380,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         body: &Body<'tcx>,
         dummy_body: &'mir Body<'tcx>,
         tcx: TyCtxt<'tcx>,
+        only_bools: bool,
     ) -> ConstPropagator<'mir, 'tcx> {
         let def_id = body.source.def_id();
         let substs = &InternalSubsts::identity_for_item(tcx, def_id);
         let param_env = tcx.param_env_reveal_all_normalized(def_id);
 
-        let can_const_prop = CanConstProp::check(tcx, param_env, body);
+        let can_const_prop = CanConstProp::check(tcx, param_env, body, only_bools);
         let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len());
         for (l, mode) in can_const_prop.iter_enumerated() {
             if *mode == ConstPropMode::OnlyInsideOwnBlock {
@@ -858,6 +866,7 @@ impl CanConstProp {
         tcx: TyCtxt<'tcx>,
         param_env: ParamEnv<'tcx>,
         body: &Body<'tcx>,
+        only_bools: bool,
     ) -> IndexVec<Local, ConstPropMode> {
         let mut cpv = CanConstProp {
             can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls),
@@ -869,6 +878,12 @@ impl CanConstProp {
         };
         for (local, val) in cpv.can_const_prop.iter_enumerated_mut() {
             let ty = body.local_decls[local].ty;
+            if only_bools {
+                if ty != tcx.types.bool {
+                    *val = ConstPropMode::NoPropagation;
+                    continue;
+                }
+            }
             match tcx.layout_of(param_env.and(ty)) {
                 Ok(layout) if layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) => {}
                 // Either the layout fails to compute, then we can't use this local anyway

compiler/rustc_mir_transform/src/const_prop_lint.rs

@@ -182,7 +182,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         let substs = &InternalSubsts::identity_for_item(tcx, def_id);
         let param_env = tcx.param_env_reveal_all_normalized(def_id);
 
-        let can_const_prop = CanConstProp::check(tcx, param_env, body);
+        let can_const_prop = CanConstProp::check(tcx, param_env, body, false);
         let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len());
         for (l, mode) in can_const_prop.iter_enumerated() {
             if *mode == ConstPropMode::OnlyInsideOwnBlock {

compiler/rustc_mir_transform/src/lib.rs

@@ -410,7 +410,7 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -
             pm::run_passes(
                 tcx,
                 &mut body,
-                &[&const_prop::ConstProp],
+                &[&const_prop::ConstProp::new(false)],
                 Some(MirPhase::Runtime(RuntimePhase::Optimized)),
             );
         }
@@ -555,6 +555,9 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &unreachable_prop::UnreachablePropagation,
             &uninhabited_enum_branching::UninhabitedEnumBranching,
             &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")),
+            &const_prop::ConstProp::new(true),
+            &const_debuginfo::ConstDebugInfo,
+            &o1(simplify_branches::SimplifyConstCondition::new("before-inline")),
             &inline::Inline,
             &remove_storage_markers::RemoveStorageMarkers,
             &remove_zsts::RemoveZsts,
@@ -568,7 +571,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &separate_const_switch::SeparateConstSwitch,
             //
             // FIXME(#70073): This pass is responsible for both optimization as well as some lints.
-            &const_prop::ConstProp,
+            &const_prop::ConstProp::new(false),
             &dataflow_const_prop::DataflowConstProp,
             //
             // Const-prop runs unconditionally, but doesn't mutate the MIR at mir-opt-level=0.

library/core/src/intrinsics.rs

@@ -2297,12 +2297,22 @@ pub(crate) use assert_unsafe_precondition;
 
 /// Checks whether `ptr` is properly aligned with respect to
 /// `align_of::<T>()`.
+#[inline]
 pub(crate) fn is_aligned_and_not_null<T>(ptr: *const T) -> bool {
-    !ptr.is_null() && ptr.is_aligned()
+    // A reasonable implementation of this would be
+    // !ptr.is_null() && ptr.is_aligned()
+    // However that implementation is based on many layers of abstraction, and results in a lot
+    // more MIR being generated, which matters for this function especially because when debug
+    // assertions are enabled it is called in very many places. This simpler implementation seems
+    // to be worth 0-5% on debug builds.
+    let addr = ptr.addr();
+    let mask = const { crate::mem::align_of::<T>() - 1 };
+    (addr != 0) && (addr & mask == 0)
 }
 
 /// Checks whether an allocation of `len` instances of `T` exceeds
 /// the maximum allowed allocation size.
+#[inline]
 pub(crate) fn is_valid_allocation_size<T>(len: usize) -> bool {
     let max_len = const {
         let size = crate::mem::size_of::<T>();
@@ -2313,6 +2323,7 @@ pub(crate) fn is_valid_allocation_size<T>(len: usize) -> bool {
 
 /// Checks whether the regions of memory starting at `src` and `dst` of size
 /// `count * size_of::<T>()` do *not* overlap.
+#[inline]
 pub(crate) fn is_nonoverlapping<T>(src: *const T, dst: *const T, count: usize) -> bool {
     let src_usize = src.addr();
     let dst_usize = dst.addr();

src/test/mir-opt/const_debuginfo.main.ConstDebugInfo.diff

@@ -11,24 +11,20 @@
       let mut _14: u32;                    // in scope 0 at $DIR/const_debuginfo.rs:+13:13: +13:16
       let mut _15: u32;                    // in scope 0 at $DIR/const_debuginfo.rs:+13:19: +13:22
       scope 1 {
--         debug x => _1;                   // in scope 1 at $DIR/const_debuginfo.rs:+1:9: +1:10
-+         debug x => const 1_u8;           // in scope 1 at $DIR/const_debuginfo.rs:+1:9: +1:10
+          debug x => const 1_u8;           // in scope 1 at $DIR/const_debuginfo.rs:+1:9: +1:10
           let _2: u8;                      // in scope 1 at $DIR/const_debuginfo.rs:+2:9: +2:10
           scope 2 {
--             debug y => _2;               // in scope 2 at $DIR/const_debuginfo.rs:+2:9: +2:10
-+             debug y => const 2_u8;       // in scope 2 at $DIR/const_debuginfo.rs:+2:9: +2:10
+              debug y => const 2_u8;       // in scope 2 at $DIR/const_debuginfo.rs:+2:9: +2:10
               let _3: u8;                  // in scope 2 at $DIR/const_debuginfo.rs:+3:9: +3:10
               scope 3 {
--                 debug z => _3;           // in scope 3 at $DIR/const_debuginfo.rs:+3:9: +3:10
-+                 debug z => const 3_u8;   // in scope 3 at $DIR/const_debuginfo.rs:+3:9: +3:10
+                  debug z => const 3_u8;   // in scope 3 at $DIR/const_debuginfo.rs:+3:9: +3:10
                   let _4: u8;              // in scope 3 at $DIR/const_debuginfo.rs:+4:9: +4:12
                   scope 4 {
 -                     debug sum => _4;     // in scope 4 at $DIR/const_debuginfo.rs:+4:9: +4:12
 +                     debug sum => const 6_u8; // in scope 4 at $DIR/const_debuginfo.rs:+4:9: +4:12
                       let _9: &str;        // in scope 4 at $DIR/const_debuginfo.rs:+6:9: +6:10
                       scope 5 {
--                         debug s => _9;   // in scope 5 at $DIR/const_debuginfo.rs:+6:9: +6:10
-+                         debug s => const "hello, world!"; // in scope 5 at $DIR/const_debuginfo.rs:+6:9: +6:10
+                          debug s => const "hello, world!"; // in scope 5 at $DIR/const_debuginfo.rs:+6:9: +6:10
                           let _10: (bool, bool, u32); // in scope 5 at $DIR/const_debuginfo.rs:+8:9: +8:10
                           let _16: bool;   // in scope 5 at $DIR/const_debuginfo.rs:+8:9: +8:10
                           let _17: bool;   // in scope 5 at $DIR/const_debuginfo.rs:+8:9: +8:10

src/test/mir-opt/const_prop/bad_op_div_by_zero.main.ConstProp.diff

@@ -31,11 +31,10 @@
 
       bb1: {
 -         _5 = Eq(_3, const -1_i32);       // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
--         _6 = Eq(const 1_i32, const i32::MIN); // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
++         _5 = const false;                // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
+          _6 = const false;                // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
 -         _7 = BitAnd(move _5, move _6);   // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
 -         assert(!move _7, "attempt to compute `{} / {}`, which would overflow", const 1_i32, _3) -> bb2; // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
-+         _5 = const false;                // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
-+         _6 = const false;                // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
 +         _7 = const false;                // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
 +         assert(!const false, "attempt to compute `{} / {}`, which would overflow", const 1_i32, _3) -> bb2; // scope 1 at $DIR/bad_op_div_by_zero.rs:+2:14: +2:19
       }

src/test/mir-opt/const_prop/bad_op_mod_by_zero.main.ConstProp.diff

@@ -32,13 +32,10 @@
 
       bb1: {
 -         _5 = Eq(_3, const -1_i32);       // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
--         _6 = Eq(const 1_i32, const i32::MIN); // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
--         _7 = BitAnd(move _5, move _6);   // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
--         assert(!move _7, "attempt to compute the remainder of `{} % {}`, which would overflow", const 1_i32, _3) -> bb2; // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
 +         _5 = const false;                // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
-+         _6 = const false;                // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
-+         _7 = const false;                // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
-+         assert(!const false, "attempt to compute the remainder of `{} % {}`, which would overflow", const 1_i32, const 0_i32) -> bb2; // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
+          _6 = const false;                // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
+          _7 = const false;                // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
+          goto -> bb2;                     // scope 1 at $DIR/bad_op_mod_by_zero.rs:+2:14: +2:19
       }
 
       bb2: {

src/test/mir-opt/const_prop/boolean_identities.test.ConstProp.diff

@@ -14,17 +14,14 @@
           StorageLive(_3);                 // scope 0 at $DIR/boolean_identities.rs:+1:5: +1:15
           StorageLive(_4);                 // scope 0 at $DIR/boolean_identities.rs:+1:6: +1:7
           _4 = _2;                         // scope 0 at $DIR/boolean_identities.rs:+1:6: +1:7
--         _3 = BitOr(move _4, const true); // scope 0 at $DIR/boolean_identities.rs:+1:5: +1:15
-+         _3 = const true;                 // scope 0 at $DIR/boolean_identities.rs:+1:5: +1:15
+          _3 = const true;                 // scope 0 at $DIR/boolean_identities.rs:+1:5: +1:15
           StorageDead(_4);                 // scope 0 at $DIR/boolean_identities.rs:+1:14: +1:15
           StorageLive(_5);                 // scope 0 at $DIR/boolean_identities.rs:+1:18: +1:29
           StorageLive(_6);                 // scope 0 at $DIR/boolean_identities.rs:+1:19: +1:20
           _6 = _1;                         // scope 0 at $DIR/boolean_identities.rs:+1:19: +1:20
--         _5 = BitAnd(move _6, const false); // scope 0 at $DIR/boolean_identities.rs:+1:18: +1:29
-+         _5 = const false;                // scope 0 at $DIR/boolean_identities.rs:+1:18: +1:29
+          _5 = const false;                // scope 0 at $DIR/boolean_identities.rs:+1:18: +1:29
           StorageDead(_6);                 // scope 0 at $DIR/boolean_identities.rs:+1:28: +1:29
--         _0 = BitAnd(move _3, move _5);   // scope 0 at $DIR/boolean_identities.rs:+1:5: +1:29
-+         _0 = const false;                // scope 0 at $DIR/boolean_identities.rs:+1:5: +1:29
+          _0 = const false;                // scope 0 at $DIR/boolean_identities.rs:+1:5: +1:29
           StorageDead(_5);                 // scope 0 at $DIR/boolean_identities.rs:+1:28: +1:29
           StorageDead(_3);                 // scope 0 at $DIR/boolean_identities.rs:+1:28: +1:29
           return;                          // scope 0 at $DIR/boolean_identities.rs:+2:2: +2:2

src/test/mir-opt/const_prop/control_flow_simplification.hello.ConstProp.diff

@@ -9,8 +9,7 @@
       bb0: {
           StorageLive(_1);                 // scope 0 at $DIR/control_flow_simplification.rs:+1:8: +1:21
           _1 = const _;                    // scope 0 at $DIR/control_flow_simplification.rs:+1:8: +1:21
--         switchInt(move _1) -> [0: bb2, otherwise: bb1]; // scope 0 at $DIR/control_flow_simplification.rs:+1:8: +1:21
-+         switchInt(const false) -> [0: bb2, otherwise: bb1]; // scope 0 at $DIR/control_flow_simplification.rs:+1:8: +1:21
+          goto -> bb2;                     // scope 0 at $DIR/control_flow_simplification.rs:+1:8: +1:21
       }
 
       bb1: {

src/test/mir-opt/dead_code_does_not_inline.main.Inline.after.mir

@@ -0,0 +1,37 @@
+// MIR for `main` after Inline
+
+fn main() -> () {
+    let mut _0: ();                      // return place in scope 0 at $DIR/dead_code_does_not_inline.rs:+0:15: +0:15
+    let mut _1: bool;                    // in scope 0 at $DIR/dead_code_does_not_inline.rs:+1:8: +1:13
+    let _2: ();                          // in scope 0 at $DIR/dead_code_does_not_inline.rs:+2:9: +2:17
+
+    bb0: {
+        StorageLive(_1);                 // scope 0 at $DIR/dead_code_does_not_inline.rs:+1:8: +1:13
+        _1 = const false;                // scope 0 at $DIR/dead_code_does_not_inline.rs:+1:8: +1:13
+        goto -> bb3;                     // scope 0 at $DIR/dead_code_does_not_inline.rs:+1:8: +1:13
+    }
+
+    bb1: {
+        StorageLive(_2);                 // scope 0 at $DIR/dead_code_does_not_inline.rs:+2:9: +2:17
+        _2 = callee() -> bb2;            // scope 0 at $DIR/dead_code_does_not_inline.rs:+2:9: +2:17
+                                         // mir::Constant
+                                         // + span: $DIR/dead_code_does_not_inline.rs:4:9: 4:15
+                                         // + literal: Const { ty: fn() {callee}, val: Value(<ZST>) }
+    }
+
+    bb2: {
+        StorageDead(_2);                 // scope 0 at $DIR/dead_code_does_not_inline.rs:+2:17: +2:18
+        _0 = const ();                   // scope 0 at $DIR/dead_code_does_not_inline.rs:+1:14: +3:6
+        goto -> bb4;                     // scope 0 at $DIR/dead_code_does_not_inline.rs:+1:5: +3:6
+    }
+
+    bb3: {
+        _0 = const ();                   // scope 0 at $DIR/dead_code_does_not_inline.rs:+3:6: +3:6
+        goto -> bb4;                     // scope 0 at $DIR/dead_code_does_not_inline.rs:+1:5: +3:6
+    }
+
+    bb4: {
+        StorageDead(_1);                 // scope 0 at $DIR/dead_code_does_not_inline.rs:+3:5: +3:6
+        return;                          // scope 0 at $DIR/dead_code_does_not_inline.rs:+4:2: +4:2
+    }
+}

src/test/mir-opt/dead_code_does_not_inline.rs

@@ -0,0 +1,10 @@
+// EMIT_MIR dead_code_does_not_inline.main.Inline.after.mir
+pub fn main() {
+    if false {
+        callee();
+    }
+}
+
+pub fn callee() {
+    println!("Wooooo I'm invisible");
+}

src/test/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.diff

@@ -9,30 +9,15 @@
       bb0: {
           StorageLive(_1);                 // scope 0 at $DIR/simplify_if.rs:+1:8: +1:13
           _1 = const false;                // scope 0 at $DIR/simplify_if.rs:+1:8: +1:13
--         switchInt(const false) -> [0: bb3, otherwise: bb1]; // scope 0 at $DIR/simplify_if.rs:+1:8: +1:13
-+         goto -> bb3;                     // scope 0 at $DIR/simplify_if.rs:+1:8: +1:13
+          goto -> bb1;                     // scope 0 at $DIR/simplify_if.rs:+1:8: +1:13
       }
 
       bb1: {
-          StorageLive(_2);                 // scope 0 at $DIR/simplify_if.rs:+2:9: +2:15
-          _2 = noop() -> bb2;              // scope 0 at $DIR/simplify_if.rs:+2:9: +2:15
-                                           // mir::Constant
-                                           // + span: $DIR/simplify_if.rs:7:9: 7:13
-                                           // + literal: Const { ty: fn() {noop}, val: Value(<ZST>) }
-      }
-
-      bb2: {
-          StorageDead(_2);                 // scope 0 at $DIR/simplify_if.rs:+2:15: +2:16
-          nop;                             // scope 0 at $DIR/simplify_if.rs:+1:14: +3:6
-          goto -> bb4;                     // scope 0 at $DIR/simplify_if.rs:+1:5: +3:6
-      }
-
-      bb3: {
           nop;                             // scope 0 at $DIR/simplify_if.rs:+3:6: +3:6
-          goto -> bb4;                     // scope 0 at $DIR/simplify_if.rs:+1:5: +3:6
+          goto -> bb2;                     // scope 0 at $DIR/simplify_if.rs:+1:5: +3:6
       }
 
-      bb4: {
+      bb2: {
           StorageDead(_1);                 // scope 0 at $DIR/simplify_if.rs:+3:5: +3:6
           return;                          // scope 0 at $DIR/simplify_if.rs:+4:2: +4:2
       }

src/test/mir-opt/simplify_match.main.ConstProp.diff

@@ -13,26 +13,17 @@
           StorageLive(_1);                 // scope 0 at $DIR/simplify_match.rs:+1:11: +1:31
           StorageLive(_2);                 // scope 0 at $DIR/simplify_match.rs:+1:17: +1:18
           _2 = const false;                // scope 0 at $DIR/simplify_match.rs:+1:21: +1:26
--         _1 = _2;                         // scope 1 at $DIR/simplify_match.rs:+1:28: +1:29
-+         _1 = const false;                // scope 1 at $DIR/simplify_match.rs:+1:28: +1:29
+          _1 = const false;                // scope 1 at $DIR/simplify_match.rs:+1:28: +1:29
           StorageDead(_2);                 // scope 0 at $DIR/simplify_match.rs:+1:30: +1:31
--         switchInt(_1) -> [0: bb1, otherwise: bb2]; // scope 0 at $DIR/simplify_match.rs:+1:5: +1:31
-+         switchInt(const false) -> [0: bb1, otherwise: bb2]; // scope 0 at $DIR/simplify_match.rs:+1:5: +1:31
+          goto -> bb1;                     // scope 0 at $DIR/simplify_match.rs:+1:5: +1:31
       }
 
       bb1: {
           nop;                             // scope 0 at $DIR/simplify_match.rs:+3:18: +3:20
-          goto -> bb3;                     // scope 0 at $DIR/simplify_match.rs:+3:18: +3:20
+          goto -> bb2;                     // scope 0 at $DIR/simplify_match.rs:+3:18: +3:20
       }
 
       bb2: {
-          _0 = noop() -> bb3;              // scope 0 at $DIR/simplify_match.rs:+2:17: +2:23
-                                           // mir::Constant
-                                           // + span: $DIR/simplify_match.rs:7:17: 7:21
-                                           // + literal: Const { ty: fn() {noop}, val: Value(<ZST>) }
-      }
-
-      bb3: {
           StorageDead(_1);                 // scope 0 at $DIR/simplify_match.rs:+5:1: +5:2
           return;                          // scope 0 at $DIR/simplify_match.rs:+5:2: +5:2
       }