Skip to content

Commit c153f3a

Browse files
saethlinsaethlin
committed
Ignore the always part of #[inline(always)] in MIR inlining
1 parent 99b3346 commit c153f3a

19 files changed

+349
-934
lines changed

‎compiler/rustc_mir_transform/src/inline.rs

+5-4
Original file line numberDiff line numberDiff line change
@@ -479,11 +479,12 @@ impl<'tcx> Inliner<'tcx> {
479479
// Abort if type validation found anything fishy.
480480
checker.validation?;
481481

482+
// N.B. We still apply our cost threshold to #[inline(always)] functions.
483+
// That attribute is often applied to very large functions that exceed LLVM's (very
484+
// generous) inlining threshold. Such functions are very poor MIR inlining candidates.
485+
// Always inlining #[inline(always)] functions in MIR, on net, slows down the compiler.
482486
let cost = checker.cost;
483-
if let InlineAttr::Always = callee_attrs.inline {
484-
debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
485-
Ok(())
486-
} else if cost <= threshold {
487+
if cost <= threshold {
487488
debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
488489
Ok(())
489490
} else {

‎src/tools/miri/tests/fail/terminate-terminator.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//@compile-flags: -Zmir-opt-level=3
1+
//@compile-flags: -Zmir-opt-level=3 -Zinline-mir-hint-threshold=1000
22
// Enable MIR inlining to ensure that `TerminatorKind::Terminate` is generated
33
// instead of just `UnwindAction::Terminate`.
44

‎tests/mir-opt/inline/asm_unwind.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
//
33
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
44
// needs-asm-support
5+
// compile-flags: -Zinline-mir-hint-threshold=1000
56
#![feature(asm_unwind)]
67

78
struct D;
@@ -10,7 +11,7 @@ impl Drop for D {
1011
fn drop(&mut self) {}
1112
}
1213

13-
#[inline(always)]
14+
#[inline]
1415
fn foo() {
1516
let _d = D;
1617
unsafe { std::arch::asm!("", options(may_unwind)) };

‎tests/mir-opt/inline/cycle.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
2+
// compile-flags: -Zinline-mir-hint-threshold=1000
23

34
// EMIT_MIR cycle.f.Inline.diff
45
#[inline(always)]

‎tests/mir-opt/inline/inline_diverging.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Tests inlining of diverging calls.
22
//
33
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
4+
// compile-flags: -Zinline-mir-hint-threshold=1000
45
#![crate_type = "lib"]
56

67
// EMIT_MIR inline_diverging.f.Inline.diff

‎tests/mir-opt/inline/inline_generator.main.Inline.panic-abort.diff

+15-15
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,30 @@
44
fn main() -> () {
55
let mut _0: ();
66
let _1: std::ops::GeneratorState<i32, bool>;
7-
let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>;
8-
let mut _3: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
9-
let mut _4: [generator@$DIR/inline_generator.rs:15:5: 15:8];
7+
let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>;
8+
let mut _3: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
9+
let mut _4: [generator@$DIR/inline_generator.rs:16:5: 16:8];
1010
+ let mut _5: bool;
1111
scope 1 {
1212
debug _r => _1;
1313
}
1414
+ scope 2 (inlined g) {
1515
+ }
16-
+ scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new) {
16+
+ scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new) {
1717
+ debug pointer => _3;
1818
+ scope 4 {
19-
+ scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new_unchecked) {
19+
+ scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new_unchecked) {
2020
+ debug pointer => _3;
2121
+ }
2222
+ }
2323
+ }
2424
+ scope 6 (inlined g::{closure#0}) {
2525
+ debug a => _5;
26-
+ let mut _6: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
26+
+ let mut _6: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
2727
+ let mut _7: u32;
2828
+ let mut _8: i32;
29-
+ let mut _9: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
30-
+ let mut _10: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
29+
+ let mut _9: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
30+
+ let mut _10: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
3131
+ }
3232

3333
bb0: {
@@ -39,18 +39,18 @@
3939
- }
4040
-
4141
- bb1: {
42-
+ _4 = [generator@$DIR/inline_generator.rs:15:5: 15:8 (#0)];
42+
+ _4 = [generator@$DIR/inline_generator.rs:16:5: 16:8 (#0)];
4343
_3 = &mut _4;
44-
- _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new(move _3) -> [return: bb2, unwind unreachable];
44+
- _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new(move _3) -> [return: bb2, unwind unreachable];
4545
- }
4646
-
4747
- bb2: {
48-
+ _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]> { pointer: move _3 };
48+
+ _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]> { pointer: move _3 };
4949
StorageDead(_3);
50-
- _1 = <[generator@$DIR/inline_generator.rs:15:5: 15:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind unreachable];
50+
- _1 = <[generator@$DIR/inline_generator.rs:16:5: 16:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind unreachable];
5151
+ StorageLive(_5);
5252
+ _5 = const false;
53-
+ _6 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
53+
+ _6 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
5454
+ _7 = discriminant((*_6));
5555
+ switchInt(move _7) -> [0: bb2, 1: bb6, 3: bb7, otherwise: bb8];
5656
}
@@ -82,7 +82,7 @@
8282
+
8383
+ bb5: {
8484
+ _1 = GeneratorState::<i32, bool>::Yielded(move _8);
85-
+ _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
85+
+ _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
8686
+ discriminant((*_9)) = 3;
8787
+ goto -> bb1;
8888
+ }
@@ -95,7 +95,7 @@
9595
+ StorageLive(_8);
9696
+ StorageDead(_8);
9797
+ _1 = GeneratorState::<i32, bool>::Complete(_5);
98-
+ _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
98+
+ _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
9999
+ discriminant((*_10)) = 1;
100100
+ goto -> bb1;
101101
+ }

‎tests/mir-opt/inline/inline_generator.main.Inline.panic-unwind.diff

+15-15
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,30 @@
44
fn main() -> () {
55
let mut _0: ();
66
let _1: std::ops::GeneratorState<i32, bool>;
7-
let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>;
8-
let mut _3: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
9-
let mut _4: [generator@$DIR/inline_generator.rs:15:5: 15:8];
7+
let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>;
8+
let mut _3: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
9+
let mut _4: [generator@$DIR/inline_generator.rs:16:5: 16:8];
1010
+ let mut _5: bool;
1111
scope 1 {
1212
debug _r => _1;
1313
}
1414
+ scope 2 (inlined g) {
1515
+ }
16-
+ scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new) {
16+
+ scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new) {
1717
+ debug pointer => _3;
1818
+ scope 4 {
19-
+ scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new_unchecked) {
19+
+ scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new_unchecked) {
2020
+ debug pointer => _3;
2121
+ }
2222
+ }
2323
+ }
2424
+ scope 6 (inlined g::{closure#0}) {
2525
+ debug a => _5;
26-
+ let mut _6: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
26+
+ let mut _6: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
2727
+ let mut _7: u32;
2828
+ let mut _8: i32;
29-
+ let mut _9: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
30-
+ let mut _10: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
29+
+ let mut _9: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
30+
+ let mut _10: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
3131
+ }
3232

3333
bb0: {
@@ -39,18 +39,18 @@
3939
- }
4040
-
4141
- bb1: {
42-
+ _4 = [generator@$DIR/inline_generator.rs:15:5: 15:8 (#0)];
42+
+ _4 = [generator@$DIR/inline_generator.rs:16:5: 16:8 (#0)];
4343
_3 = &mut _4;
44-
- _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new(move _3) -> [return: bb2, unwind: bb4];
44+
- _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new(move _3) -> [return: bb2, unwind: bb4];
4545
- }
4646
-
4747
- bb2: {
48-
+ _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]> { pointer: move _3 };
48+
+ _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]> { pointer: move _3 };
4949
StorageDead(_3);
50-
- _1 = <[generator@$DIR/inline_generator.rs:15:5: 15:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind: bb4];
50+
- _1 = <[generator@$DIR/inline_generator.rs:16:5: 16:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind: bb4];
5151
+ StorageLive(_5);
5252
+ _5 = const false;
53-
+ _6 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
53+
+ _6 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
5454
+ _7 = discriminant((*_6));
5555
+ switchInt(move _7) -> [0: bb3, 1: bb7, 3: bb8, otherwise: bb9];
5656
}
@@ -87,7 +87,7 @@
8787
+
8888
+ bb6: {
8989
+ _1 = GeneratorState::<i32, bool>::Yielded(move _8);
90-
+ _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
90+
+ _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
9191
+ discriminant((*_9)) = 3;
9292
+ goto -> bb1;
9393
+ }
@@ -100,7 +100,7 @@
100100
+ StorageLive(_8);
101101
+ StorageDead(_8);
102102
+ _1 = GeneratorState::<i32, bool>::Complete(_5);
103-
+ _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
103+
+ _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
104104
+ discriminant((*_10)) = 1;
105105
+ goto -> bb1;
106106
+ }

‎tests/mir-opt/inline/inline_generator.rs

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
2+
// compile-flags: -Zinline-mir-hint-threshold=1000
23
#![feature(generators, generator_trait)]
34

45
use std::ops::Generator;
@@ -9,8 +10,8 @@ fn main() {
910
let _r = Pin::new(&mut g()).resume(false);
1011
}
1112

12-
#[inline(always)]
13+
#[inline]
1314
pub fn g() -> impl Generator<bool> {
14-
#[inline(always)]
15+
#[inline]
1516
|a| { yield if a { 7 } else { 13 } }
1617
}

0 commit comments

Comments
 (0)