Skip to content

Commit d5a03e8

Browse files
scottmcmscottmcm
committed
Allow inlining all the drops
1 parent 17c8a4e commit d5a03e8

File tree

30 files changed

+546
-267
lines changed

30 files changed

+546
-267
lines changed

‎compiler/rustc_mir_transform/src/inline.rs

+90-7
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,16 @@ use rustc_attr::InlineAttr;
77
use rustc_hir::def::DefKind;
88
use rustc_hir::def_id::DefId;
99
use rustc_index::bit_set::BitSet;
10-
use rustc_index::Idx;
10+
use rustc_index::{Idx, IndexVec};
1111
use rustc_middle::bug;
1212
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
1313
use rustc_middle::mir::visit::*;
1414
use rustc_middle::mir::*;
1515
use rustc_middle::ty::{
16-
self, Instance, InstanceKind, ParamEnv, Ty, TyCtxt, TypeFlags, TypeVisitableExt,
16+
self, GenericArg, Instance, InstanceKind, ParamEnv, Ty, TyCtxt, TypeFlags, TypeVisitableExt,
1717
};
1818
use rustc_session::config::{DebugInfo, OptLevel};
19-
use rustc_span::source_map::Spanned;
19+
use rustc_span::source_map::{dummy_spanned, Spanned};
2020
use rustc_span::sym;
2121
use rustc_target::abi::FieldIdx;
2222
use rustc_target::spec::abi::Abi;
@@ -94,6 +94,10 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
9494

9595
let param_env = tcx.param_env_reveal_all_normalized(def_id);
9696
let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
97+
let Some(drop_in_place_fn) = tcx.lang_items().drop_in_place_fn() else {
98+
error!("No `drop_in_place` function; not even trying to inline!");
99+
return false;
100+
};
97101

98102
let mut this = Inliner {
99103
tcx,
@@ -105,6 +109,8 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
105109
codegen_fn_attrs.inline,
106110
InlineAttr::Hint | InlineAttr::Always
107111
) && body_is_forwarder(body),
112+
drop_in_place_fn,
113+
unit_local: None,
108114
};
109115
let blocks = START_BLOCK..body.basic_blocks.next_index();
110116
this.process_blocks(body, blocks);
@@ -127,9 +133,78 @@ struct Inliner<'tcx> {
127133
/// Indicates that the caller is #[inline] and just calls another function,
128134
/// and thus we can inline less into it as it'll be inlined itself.
129135
caller_is_inline_forwarder: bool,
136+
/// The compiler-magic function that actually drops something.
137+
drop_in_place_fn: DefId,
138+
/// When lowering `Drop(place)` to `drop_in_place(&place)`, we need a unit
139+
/// local to use as the target of the call, but don't want multiple.
140+
unit_local: Option<Local>,
130141
}
131142

132143
impl<'tcx> Inliner<'tcx> {
144+
fn lower_drop_to_call(
145+
&mut self,
146+
block: &mut BasicBlockData<'tcx>,
147+
local_decls: &mut IndexVec<Local, LocalDecl<'tcx>>,
148+
) {
149+
let terminator = block.terminator.as_mut().unwrap();
150+
let TerminatorKind::Drop { place: dropped_place, target, unwind, replace: _ } =
151+
terminator.kind
152+
else {
153+
return;
154+
};
155+
156+
let dropped_ty = dropped_place.ty(local_decls, self.tcx).ty;
157+
if matches!(dropped_ty.kind(), ty::Alias(..) | ty::Param(..)) {
158+
// Not worth the extra locals, since we'll probably not be able to
159+
// get MIR for it. If something non-generic happens to inline this
160+
// block later, we'll have the opportunity to handle it then.
161+
return;
162+
}
163+
164+
if !dropped_ty.needs_drop(self.tcx, self.param_env) {
165+
// Leave it for other passes to remove, which is cheaper than
166+
// doing the work to inline an empty shim.
167+
return;
168+
}
169+
170+
self.changed = true;
171+
172+
let dropped_operand = if let [PlaceElem::Deref] = **dropped_place.projection
173+
&& local_decls[dropped_place.local].ty.is_mutable_ptr()
174+
{
175+
Operand::Copy(Place::from(dropped_place.local))
176+
} else {
177+
let dropped_ty_ptr = Ty::new_mut_ptr(self.tcx, dropped_ty);
178+
let ptr_local =
179+
local_decls.push(LocalDecl::new(dropped_ty_ptr, terminator.source_info.span));
180+
block.statements.push(Statement {
181+
source_info: terminator.source_info,
182+
kind: StatementKind::Assign(Box::new((
183+
Place::from(ptr_local),
184+
Rvalue::AddressOf(Mutability::Mut, dropped_place),
185+
))),
186+
});
187+
Operand::Move(Place::from(ptr_local))
188+
};
189+
let unit_local = *self.unit_local.get_or_insert_with(|| {
190+
local_decls.push(LocalDecl::new(self.tcx.types.unit, terminator.source_info.span))
191+
});
192+
terminator.kind = TerminatorKind::Call {
193+
func: Operand::function_handle(
194+
self.tcx,
195+
self.drop_in_place_fn,
196+
[GenericArg::from(dropped_ty)],
197+
terminator.source_info.span,
198+
),
199+
args: Box::new([dummy_spanned(dropped_operand)]),
200+
destination: Place::from(unit_local),
201+
target: Some(target),
202+
unwind,
203+
call_source: CallSource::Misc,
204+
fn_span: terminator.source_info.span,
205+
};
206+
}
207+
133208
fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range<BasicBlock>) {
134209
// How many callsites in this body are we allowed to inline? We need to limit this in order
135210
// to prevent super-linear growth in MIR size
@@ -140,12 +215,18 @@ impl<'tcx> Inliner<'tcx> {
140215
};
141216
let mut inlined_count = 0;
142217
for bb in blocks {
143-
let bb_data = &caller_body[bb];
144-
if bb_data.is_cleanup {
218+
if caller_body[bb].is_cleanup {
145219
continue;
146220
}
147221

148-
let Some(callsite) = self.resolve_callsite(caller_body, bb, bb_data) else {
222+
// Changing `Drop` to `Call` actually preserves the CFG because it
223+
// keeps the same `target` and `unwind` action.
224+
self.lower_drop_to_call(
225+
&mut caller_body.basic_blocks.as_mut_preserves_cfg()[bb],
226+
&mut caller_body.local_decls,
227+
);
228+
229+
let Some(callsite) = self.resolve_callsite(caller_body, bb, &caller_body[bb]) else {
149230
continue;
150231
};
151232

@@ -509,7 +590,9 @@ impl<'tcx> Inliner<'tcx> {
509590
return Err("Body is tainted");
510591
}
511592

512-
let mut threshold = if self.caller_is_inline_forwarder {
593+
let mut threshold = if self.caller_is_inline_forwarder
594+
|| matches!(callee_body.source.instance, InstanceKind::DropGlue(..))
595+
{
513596
self.tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)
514597
} else if cross_crate_inlinable {
515598
self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)

‎tests/codegen/drop-in-place-noalias.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
//@ compile-flags: -O -C no-prepopulate-passes
1+
//@ compile-flags: -O -C no-prepopulate-passes -Z inline-mir=no
22

33
// Tests that the compiler can apply `noalias` and other &mut attributes to `drop_in_place`.
44
// Note that non-Unpin types should not get `noalias`, matching &mut behavior.
55

6+
// Needs to not run MIR inlining or else everything inlines away.
7+
68
#![crate_type = "lib"]
79

810
use std::marker::PhantomPinned;

‎tests/codegen/drop.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
//@ needs-unwind - this test verifies the amount of drop calls when unwinding is used
2-
//@ compile-flags: -C no-prepopulate-passes
2+
//@ compile-flags: -C no-prepopulate-passes -Z inline-mir=no
3+
4+
// Needs to disable MIR inlining or else some of the calls are directly to
5+
// `<SomeUniqueName as Drop>::drop` (rather than via `drop_in_place`).
36

47
#![crate_type = "lib"]
58

‎tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff

+18-11
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
let mut _0: ();
66
let _1: A;
77
let mut _2: std::boxed::Box<[bool]>;
8+
let mut _12: *mut A;
9+
let mut _13: ();
10+
let mut _14: *mut std::boxed::Box<[bool]>;
811
scope 1 {
912
debug a => _1;
1013
}
@@ -39,6 +42,8 @@
3942
}
4043
}
4144
}
45+
scope 13 (inlined std::ptr::drop_in_place::<A> - shim(Some(A))) {
46+
}
4247

4348
bb0: {
4449
StorageLive(_1);
@@ -55,26 +60,21 @@
5560
StorageLive(_11);
5661
StorageLive(_8);
5762
_8 = UbChecks();
58-
switchInt(move _8) -> [0: bb4, otherwise: bb2];
63+
switchInt(move _8) -> [0: bb3, otherwise: bb1];
5964
}
6065

6166
bb1: {
62-
StorageDead(_1);
63-
return;
64-
}
65-
66-
bb2: {
6767
StorageLive(_10);
6868
_10 = const {0x1 as *mut ()};
69-
_9 = NonNull::<T>::new_unchecked::precondition_check(const {0x1 as *mut ()}) -> [return: bb3, unwind unreachable];
69+
_9 = NonNull::<T>::new_unchecked::precondition_check(const {0x1 as *mut ()}) -> [return: bb2, unwind unreachable];
7070
}
7171

72-
bb3: {
72+
bb2: {
7373
StorageDead(_10);
74-
goto -> bb4;
74+
goto -> bb3;
7575
}
7676

77-
bb4: {
77+
bb3: {
7878
StorageDead(_8);
7979
_11 = const {0x1 as *const [bool; 0]};
8080
_5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
@@ -90,7 +90,14 @@
9090
_1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
9191
StorageDead(_2);
9292
_0 = const ();
93-
drop(_1) -> [return: bb1, unwind unreachable];
93+
_12 = &raw mut _1;
94+
_14 = &raw mut ((*_12).0: std::boxed::Box<[bool]>);
95+
_13 = std::ptr::drop_in_place::<Box<[bool]>>(move _14) -> [return: bb4, unwind unreachable];
96+
}
97+
98+
bb4: {
99+
StorageDead(_1);
100+
return;
94101
}
95102
}
96103

‎tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff

+20-13
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
let mut _0: ();
66
let _1: A;
77
let mut _2: std::boxed::Box<[bool]>;
8+
let mut _12: *mut A;
9+
let mut _13: ();
10+
let mut _14: *mut std::boxed::Box<[bool]>;
811
scope 1 {
912
debug a => _1;
1013
}
@@ -39,6 +42,8 @@
3942
}
4043
}
4144
}
45+
scope 13 (inlined std::ptr::drop_in_place::<A> - shim(Some(A))) {
46+
}
4247

4348
bb0: {
4449
StorageLive(_1);
@@ -55,30 +60,25 @@
5560
StorageLive(_11);
5661
StorageLive(_8);
5762
_8 = UbChecks();
58-
switchInt(move _8) -> [0: bb5, otherwise: bb3];
59-
}
60-
61-
bb1: {
62-
StorageDead(_1);
63-
return;
63+
switchInt(move _8) -> [0: bb4, otherwise: bb2];
6464
}
6565

66-
bb2 (cleanup): {
66+
bb1 (cleanup): {
6767
resume;
6868
}
6969

70-
bb3: {
70+
bb2: {
7171
StorageLive(_10);
7272
_10 = const {0x1 as *mut ()};
73-
_9 = NonNull::<T>::new_unchecked::precondition_check(const {0x1 as *mut ()}) -> [return: bb4, unwind unreachable];
73+
_9 = NonNull::<T>::new_unchecked::precondition_check(const {0x1 as *mut ()}) -> [return: bb3, unwind unreachable];
7474
}
7575

76-
bb4: {
76+
bb3: {
7777
StorageDead(_10);
78-
goto -> bb5;
78+
goto -> bb4;
7979
}
8080

81-
bb5: {
81+
bb4: {
8282
StorageDead(_8);
8383
_11 = const {0x1 as *const [bool; 0]};
8484
_5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
@@ -94,7 +94,14 @@
9494
_1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
9595
StorageDead(_2);
9696
_0 = const ();
97-
drop(_1) -> [return: bb1, unwind: bb2];
97+
_12 = &raw mut _1;
98+
_14 = &raw mut ((*_12).0: std::boxed::Box<[bool]>);
99+
_13 = std::ptr::drop_in_place::<Box<[bool]>>(move _14) -> [return: bb5, unwind: bb1];
100+
}
101+
102+
bb5: {
103+
StorageDead(_1);
104+
return;
98105
}
99106
}
100107

‎tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff

+18-11
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
let mut _0: ();
66
let _1: A;
77
let mut _2: std::boxed::Box<[bool]>;
8+
let mut _12: *mut A;
9+
let mut _13: ();
10+
let mut _14: *mut std::boxed::Box<[bool]>;
811
scope 1 {
912
debug a => _1;
1013
}
@@ -39,6 +42,8 @@
3942
}
4043
}
4144
}
45+
scope 13 (inlined std::ptr::drop_in_place::<A> - shim(Some(A))) {
46+
}
4247

4348
bb0: {
4449
StorageLive(_1);
@@ -55,26 +60,21 @@
5560
StorageLive(_11);
5661
StorageLive(_8);
5762
_8 = UbChecks();
58-
switchInt(move _8) -> [0: bb4, otherwise: bb2];
63+
switchInt(move _8) -> [0: bb3, otherwise: bb1];
5964
}
6065

6166
bb1: {
62-
StorageDead(_1);
63-
return;
64-
}
65-
66-
bb2: {
6767
StorageLive(_10);
6868
_10 = const {0x1 as *mut ()};
69-
_9 = NonNull::<T>::new_unchecked::precondition_check(const {0x1 as *mut ()}) -> [return: bb3, unwind unreachable];
69+
_9 = NonNull::<T>::new_unchecked::precondition_check(const {0x1 as *mut ()}) -> [return: bb2, unwind unreachable];
7070
}
7171

72-
bb3: {
72+
bb2: {
7373
StorageDead(_10);
74-
goto -> bb4;
74+
goto -> bb3;
7575
}
7676

77-
bb4: {
77+
bb3: {
7878
StorageDead(_8);
7979
_11 = const {0x1 as *const [bool; 0]};
8080
_5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
@@ -90,7 +90,14 @@
9090
_1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
9191
StorageDead(_2);
9292
_0 = const ();
93-
drop(_1) -> [return: bb1, unwind unreachable];
93+
_12 = &raw mut _1;
94+
_14 = &raw mut ((*_12).0: std::boxed::Box<[bool]>);
95+
_13 = std::ptr::drop_in_place::<Box<[bool]>>(move _14) -> [return: bb4, unwind unreachable];
96+
}
97+
98+
bb4: {
99+
StorageDead(_1);
100+
return;
94101
}
95102
}
96103

0 commit comments

Comments
 (0)