Skip to content

Commit 55eefde

Browse files
saethlinsaethlin
committed
Enable MIR inlining of very small functions in debug builds
1 parent 2a7c2df commit 55eefde

25 files changed

+160
-44
lines changed

Diff for: ‎compiler/rustc_mir_transform/src/inline.rs

+21-8
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ use rustc_middle::mir::visit::*;
1010
use rustc_middle::mir::*;
1111
use rustc_middle::ty::TypeVisitableExt;
1212
use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
13-
use rustc_session::config::OptLevel;
1413
use rustc_target::abi::FieldIdx;
1514
use rustc_target::spec::abi::Abi;
1615

@@ -47,12 +46,8 @@ impl<'tcx> MirPass<'tcx> for Inline {
4746
}
4847

4948
match sess.mir_opt_level() {
50-
0 | 1 => false,
51-
2 => {
52-
(sess.opts.optimize == OptLevel::Default
53-
|| sess.opts.optimize == OptLevel::Aggressive)
54-
&& sess.opts.incremental == None
55-
}
49+
0 => false,
50+
1 | 2 => sess.opts.incremental.is_none(),
5651
_ => true,
5752
}
5853
}
@@ -471,7 +466,15 @@ impl<'tcx> Inliner<'tcx> {
471466
if callee_body.basic_blocks.len() <= 3 {
472467
threshold += threshold / 4;
473468
}
474-
debug!(" final inline threshold = {}", threshold);
469+
470+
if tcx.sess.mir_opt_level() == 1 {
471+
threshold = 2 * INSTR_COST;
472+
473+
if callee_body.basic_blocks.len() > 1 {
474+
return Err("cost above threshold");
475+
}
476+
}
477+
debug!("final inline threshold = {}", threshold);
475478

476479
// FIXME: Give a bonus to functions with only a single caller
477480

@@ -481,6 +484,7 @@ impl<'tcx> Inliner<'tcx> {
481484
instance: callsite.callee,
482485
callee_body,
483486
cost: 0,
487+
threshold,
484488
};
485489

486490
// Traverse the MIR manually so we can account for the effects of inlining on the CFG.
@@ -493,6 +497,10 @@ impl<'tcx> Inliner<'tcx> {
493497

494498
let blk = &callee_body.basic_blocks[bb];
495499
checker.visit_basic_block_data(bb, blk);
500+
if callee_attrs.inline != InlineAttr::Always && checker.cost > threshold {
501+
debug!("checker cost exceeded threshold partway through");
502+
return Err("cost above threshold");
503+
}
496504

497505
let term = blk.terminator();
498506
if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind {
@@ -805,12 +813,17 @@ struct CostChecker<'b, 'tcx> {
805813
tcx: TyCtxt<'tcx>,
806814
param_env: ParamEnv<'tcx>,
807815
cost: usize,
816+
threshold: usize,
808817
callee_body: &'b Body<'tcx>,
809818
instance: ty::Instance<'tcx>,
810819
}
811820

812821
impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
813822
fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
823+
if self.cost > self.threshold {
824+
return;
825+
}
826+
814827
// Don't count StorageLive/StorageDead in the inlining cost.
815828
match statement.kind {
816829
StatementKind::StorageLive(_)

Diff for: ‎tests/codegen-units/polymorphization/unused_type_parameters.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1
1+
// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1 -Zmir-opt-level=0
22

33
#![crate_type = "rlib"]
44

Diff for: ‎tests/debuginfo/function-names.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// Function names are formatted differently in old versions of GDB
22
// min-gdb-version: 10.1
33

4-
// compile-flags:-g
4+
// compile-flags: -g -Zmir-opt-level=0
55

66
// === GDB TESTS ===================================================================================
77

Diff for: ‎tests/mir-opt/inline/trivial_fn_debug_build.rs

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
2+
// compile-flags: -Copt-level=0 -Zmir-opt-level=1
3+
#![crate_type = "lib"]
4+
5+
// Test that we still inline trivial functions even in a debug build
6+
7+
pub struct Thing {
8+
inner: u8,
9+
}
10+
11+
impl Thing {
12+
#[inline]
13+
pub fn get(&self) -> u8 {
14+
self.inner
15+
}
16+
}
17+
18+
// EMIT_MIR trivial_fn_debug_build.wrapper.Inline.diff
19+
// EMIT_MIR trivial_fn_debug_build.wrapper.PreCodegen.after.mir
20+
pub fn wrapper(t: &Thing) -> u8 {
21+
t.get()
22+
}

Diff for: ‎tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-abort.diff

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
- // MIR for `wrapper` before Inline
2+
+ // MIR for `wrapper` after Inline
3+
4+
fn wrapper(_1: &Thing) -> u8 {
5+
debug t => _1;
6+
let mut _0: u8;
7+
let mut _2: &Thing;
8+
+ scope 1 (inlined Thing::get) {
9+
+ debug self => _2;
10+
+ }
11+
12+
bb0: {
13+
StorageLive(_2);
14+
_2 = &(*_1);
15+
- _0 = Thing::get(move _2) -> [return: bb1, unwind unreachable];
16+
- }
17+
-
18+
- bb1: {
19+
+ _0 = ((*_2).0: u8);
20+
StorageDead(_2);
21+
return;
22+
}
23+
}
24+

Diff for: ‎tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-unwind.diff

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
- // MIR for `wrapper` before Inline
2+
+ // MIR for `wrapper` after Inline
3+
4+
fn wrapper(_1: &Thing) -> u8 {
5+
debug t => _1;
6+
let mut _0: u8;
7+
let mut _2: &Thing;
8+
+ scope 1 (inlined Thing::get) {
9+
+ debug self => _2;
10+
+ }
11+
12+
bb0: {
13+
StorageLive(_2);
14+
_2 = &(*_1);
15+
- _0 = Thing::get(move _2) -> [return: bb1, unwind continue];
16+
- }
17+
-
18+
- bb1: {
19+
+ _0 = ((*_2).0: u8);
20+
StorageDead(_2);
21+
return;
22+
}
23+
}
24+

Diff for: ‎tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-abort.mir

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// MIR for `wrapper` after PreCodegen
2+
3+
fn wrapper(_1: &Thing) -> u8 {
4+
debug t => _1;
5+
let mut _0: u8;
6+
scope 1 (inlined Thing::get) {
7+
debug self => _1;
8+
}
9+
10+
bb0: {
11+
_0 = ((*_1).0: u8);
12+
return;
13+
}
14+
}

Diff for: ‎tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-unwind.mir

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// MIR for `wrapper` after PreCodegen
2+
3+
fn wrapper(_1: &Thing) -> u8 {
4+
debug t => _1;
5+
let mut _0: u8;
6+
scope 1 (inlined Thing::get) {
7+
debug self => _1;
8+
}
9+
10+
bb0: {
11+
_0 = ((*_1).0: u8);
12+
return;
13+
}
14+
}

Diff for: ‎tests/run-make/sepcomp-cci-copies/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ include ../tools.mk
88
all:
99
$(RUSTC) cci_lib.rs
1010
$(RUSTC) foo.rs --emit=llvm-ir -C codegen-units=6 \
11-
-Z inline-in-all-cgus
11+
-Z inline-in-all-cgus -Zmir-opt-level=0
1212
[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ .*cci_fn)" -eq "2" ]

Diff for: ‎tests/run-make/sepcomp-inlining/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ include ../tools.mk
88

99
all:
1010
$(RUSTC) foo.rs --emit=llvm-ir -C codegen-units=3 \
11-
-Z inline-in-all-cgus
11+
-Z inline-in-all-cgus -Zmir-opt-level=0
1212
[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ i32\ .*inlined)" -eq "0" ]
1313
[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ internal\ i32\ .*inlined)" -eq "2" ]
1414
[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ hidden\ i32\ .*normal)" -eq "1" ]

Diff for: ‎tests/run-make/symbol-visibility/Makefile

+11-11
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,12 @@ endif
3232
RE_ANY_RUST_SYMBOL="_ZN.*h.*E\|_R[a-zA-Z0-9_]+"
3333

3434
all:
35-
$(RUSTC) -Zshare-generics=no an_rlib.rs
36-
$(RUSTC) -Zshare-generics=no a_cdylib.rs
37-
$(RUSTC) -Zshare-generics=no a_rust_dylib.rs
38-
$(RUSTC) -Zshare-generics=no a_proc_macro.rs
39-
$(RUSTC) -Zshare-generics=no an_executable.rs
40-
$(RUSTC) -Zshare-generics=no a_cdylib.rs --crate-name combined_rlib_dylib --crate-type=rlib,cdylib
35+
$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 an_rlib.rs
36+
$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_cdylib.rs
37+
$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_rust_dylib.rs
38+
$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_proc_macro.rs
39+
$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 an_executable.rs
40+
$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_cdylib.rs --crate-name combined_rlib_dylib --crate-type=rlib,cdylib
4141

4242
# Check that a cdylib exports its public #[no_mangle] functions
4343
[ "$$($(NM) $(TMPDIR)/$(CDYLIB_NAME) | grep -v __imp_ | grep -c public_c_function_from_cdylib)" -eq "1" ]
@@ -85,11 +85,11 @@ endif
8585
[ "$$($(NM) $(TMPDIR)/$(COMBINED_CDYLIB_NAME) | grep -v __imp_ | grep -c $(RE_ANY_RUST_SYMBOL))" -eq "0" ]
8686

8787

88-
$(RUSTC) -Zshare-generics=yes an_rlib.rs
89-
$(RUSTC) -Zshare-generics=yes a_cdylib.rs
90-
$(RUSTC) -Zshare-generics=yes a_rust_dylib.rs
91-
$(RUSTC) -Zshare-generics=yes a_proc_macro.rs
92-
$(RUSTC) -Zshare-generics=yes an_executable.rs
88+
$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 an_rlib.rs
89+
$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_cdylib.rs
90+
$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_rust_dylib.rs
91+
$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_proc_macro.rs
92+
$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 an_executable.rs
9393

9494
# Check that a cdylib exports its public #[no_mangle] functions
9595
[ "$$($(NM) $(TMPDIR)/$(CDYLIB_NAME) | grep -v __imp_ | grep -c public_c_function_from_cdylib)" -eq "1" ]

Diff for: ‎tests/ui-fulldeps/stable-mir/crate-info.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ fn test_stable_mir(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
5959
let foo_bar = get_item(&items, (DefKind::Fn, "foo_bar")).unwrap();
6060
let body = foo_bar.body();
6161
assert_eq!(body.locals.len(), 5);
62-
assert_eq!(body.blocks.len(), 4);
62+
assert_eq!(body.blocks.len(), 3);
6363
let block = &body.blocks[0];
6464
match &block.terminator.kind {
6565
stable_mir::mir::TerminatorKind::Call { .. } => {}

Diff for: ‎tests/ui/consts/const-eval/index-out-of-bounds-never-type.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// build-fail
2+
// compile-flags: -Zmir-opt-level=0
23

34
// Regression test for #66975
45
#![warn(unconditional_panic)]

Diff for: ‎tests/ui/consts/const-eval/index-out-of-bounds-never-type.stderr

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
error[E0080]: evaluation of `PrintName::<()>::VOID` failed
2-
--> $DIR/index-out-of-bounds-never-type.rs:10:61
2+
--> $DIR/index-out-of-bounds-never-type.rs:11:61
33
|
44
LL | const VOID: ! = { let x = 0 * std::mem::size_of::<T>(); [][x] };
55
| ^^^^^ index out of bounds: the length is 0 but the index is 0
66

77
note: the above error was encountered while instantiating `fn f::<()>`
8-
--> $DIR/index-out-of-bounds-never-type.rs:20:5
8+
--> $DIR/index-out-of-bounds-never-type.rs:21:5
99
|
1010
LL | f::<()>();
1111
| ^^^^^^^^^

Diff for: ‎tests/ui/consts/const-eval/issue-50814-2.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// build-fail
2+
// compile-flags: -Zmir-opt-level=0
23

34
trait C {
45
const BOO: usize;

Diff for: ‎tests/ui/consts/const-eval/issue-50814-2.stderr

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
error[E0080]: evaluation of `<A<()> as Foo<()>>::BAR` failed
2-
--> $DIR/issue-50814-2.rs:14:24
2+
--> $DIR/issue-50814-2.rs:15:24
33
|
44
LL | const BAR: usize = [5, 6, 7][T::BOO];
55
| ^^^^^^^^^^^^^^^^^ index out of bounds: the length is 3 but the index is 42
66

77
note: erroneous constant encountered
8-
--> $DIR/issue-50814-2.rs:18:6
8+
--> $DIR/issue-50814-2.rs:19:6
99
|
1010
LL | &<A<T> as Foo<T>>::BAR
1111
| ^^^^^^^^^^^^^^^^^^^^^
1212

1313
note: the above error was encountered while instantiating `fn foo::<()>`
14-
--> $DIR/issue-50814-2.rs:30:22
14+
--> $DIR/issue-50814-2.rs:31:22
1515
|
1616
LL | println!("{:x}", foo::<()>() as *const usize as usize);
1717
| ^^^^^^^^^^^

Diff for: ‎tests/ui/consts/const-eval/issue-85155.rs

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
// aux-build: post_monomorphization_error.rs
1212
// build-fail: this is a post-monomorphization error, it passes check runs and requires building
1313
// to actually fail.
14+
// compile-flags: -Zmir-opt-level=0
1415

1516
extern crate post_monomorphization_error;
1617

Diff for: ‎tests/ui/consts/const-eval/issue-85155.stderr

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ LL | let _ = 1 / ((IMM >= MIN && IMM <= MAX) as usize);
55
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ attempt to divide `1_usize` by zero
66

77
note: the above error was encountered while instantiating `fn post_monomorphization_error::stdarch_intrinsic::<2>`
8-
--> $DIR/issue-85155.rs:19:5
8+
--> $DIR/issue-85155.rs:20:5
99
|
1010
LL | post_monomorphization_error::stdarch_intrinsic::<2>();
1111
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Diff for: ‎tests/ui/generics/post_monomorphization_error_backtrace.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// build-fail
2+
// compile-flags: -Zmir-opt-level=0
23

34
fn assert_zst<T>() {
45
struct F<T>(T);

Diff for: ‎tests/ui/generics/post_monomorphization_error_backtrace.stderr

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,27 @@
11
error[E0080]: evaluation of `assert_zst::F::<u32>::V` failed
2-
--> $DIR/post_monomorphization_error_backtrace.rs:6:23
2+
--> $DIR/post_monomorphization_error_backtrace.rs:7:23
33
|
44
LL | const V: () = assert!(std::mem::size_of::<T>() == 0);
5-
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:6:23
5+
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:7:23
66
|
77
= note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
88

99
note: the above error was encountered while instantiating `fn assert_zst::<u32>`
10-
--> $DIR/post_monomorphization_error_backtrace.rs:18:5
10+
--> $DIR/post_monomorphization_error_backtrace.rs:19:5
1111
|
1212
LL | assert_zst::<U>()
1313
| ^^^^^^^^^^^^^^^^^
1414

1515
error[E0080]: evaluation of `assert_zst::F::<i32>::V` failed
16-
--> $DIR/post_monomorphization_error_backtrace.rs:6:23
16+
--> $DIR/post_monomorphization_error_backtrace.rs:7:23
1717
|
1818
LL | const V: () = assert!(std::mem::size_of::<T>() == 0);
19-
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:6:23
19+
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:7:23
2020
|
2121
= note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
2222

2323
note: the above error was encountered while instantiating `fn assert_zst::<i32>`
24-
--> $DIR/post_monomorphization_error_backtrace.rs:18:5
24+
--> $DIR/post_monomorphization_error_backtrace.rs:19:5
2525
|
2626
LL | assert_zst::<U>()
2727
| ^^^^^^^^^^^^^^^^^

Diff for: ‎tests/ui/inline-const/const-expr-generic-err.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// build-fail
2+
// compile-flags: -Zmir-opt-level=0
23
#![feature(inline_const)]
34

45
fn foo<T>() {

Diff for: ‎tests/ui/inline-const/const-expr-generic-err.stderr

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
11
error[E0080]: evaluation of `foo::<i32>::{constant#0}` failed
2-
--> $DIR/const-expr-generic-err.rs:5:13
2+
--> $DIR/const-expr-generic-err.rs:6:13
33
|
44
LL | const { assert!(std::mem::size_of::<T>() == 0); }
5-
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/const-expr-generic-err.rs:5:13
5+
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/const-expr-generic-err.rs:6:13
66
|
77
= note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
88

99
note: the above error was encountered while instantiating `fn foo::<i32>`
10-
--> $DIR/const-expr-generic-err.rs:13:5
10+
--> $DIR/const-expr-generic-err.rs:14:5
1111
|
1212
LL | foo::<i32>();
1313
| ^^^^^^^^^^^^
1414

1515
error[E0080]: evaluation of `bar::<0>::{constant#0}` failed
16-
--> $DIR/const-expr-generic-err.rs:9:13
16+
--> $DIR/const-expr-generic-err.rs:10:13
1717
|
1818
LL | const { N - 1 }
1919
| ^^^^^ attempt to compute `0_usize - 1_usize`, which would overflow
2020

2121
note: the above error was encountered while instantiating `fn bar::<0>`
22-
--> $DIR/const-expr-generic-err.rs:14:5
22+
--> $DIR/const-expr-generic-err.rs:15:5
2323
|
2424
LL | bar::<0>();
2525
| ^^^^^^^^^^

0 commit comments

Comments
 (0)