rust-lang · workingjubilee · Sep 21, 2024 · Sep 22, 2024 · Sep 22, 2024
diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
@@ -46,6 +46,7 @@ pub enum InlineAttr {
     Hint,
     Always,
     Never,
+    Usually,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs
@@ -31,6 +31,7 @@ fn inline_attr<'gcc, 'tcx>(
             }
         }
         InlineAttr::None => None,
+        InlineAttr::Usually => None,
     }
 }
 

diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -47,6 +47,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
             }
         }
         InlineAttr::None => None,
+        InlineAttr::Usually => {
+            Some(llvm::CreateAttrStringValue(cx.llcx, "function-inline-cost", "0"))
+        }
     }
 }
 

diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -546,6 +546,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                     InlineAttr::Always
                 } else if list_contains_name(items, sym::never) {
                     InlineAttr::Never
+                } else if list_contains_name(items, sym::usually) {
+                    InlineAttr::Usually
                 } else {
                     struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
                         .with_help("valid inline arguments are `always` and `never`")

diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
@@ -138,7 +138,7 @@ impl<'tcx> MonoItem<'tcx> {
                 // conflict with upstream crates as it could be an exported
                 // symbol.
                 match tcx.codegen_fn_attrs(instance.def_id()).inline {
-                    InlineAttr::Always => InstantiationMode::LocalCopy,
+                    InlineAttr::Always | InlineAttr::Usually => InstantiationMode::LocalCopy,
                     _ => InstantiationMode::GloballyShared { may_conflict: true },
                 }
             }

diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always => return true,
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually => return true,
         _ => {}
     }
 

diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
@@ -106,7 +106,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
         changed: false,
         caller_is_inline_forwarder: matches!(
             codegen_fn_attrs.inline,
-            InlineAttr::Hint | InlineAttr::Always
+            InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually
         ) && body_is_forwarder(body),
     };
     let blocks = START_BLOCK..body.basic_blocks.next_index();

diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
@@ -2101,6 +2101,7 @@ symbols! {
         usize_legacy_fn_max_value,
         usize_legacy_fn_min_value,
         usize_legacy_mod,
+        usually,
         va_arg,
         va_copy,
         va_end,

diff --git a/library/alloc/src/boxed.rs b/library/alloc/src/boxed.rs
@@ -246,7 +246,7 @@ impl<T> Box<T> {
     /// let five = Box::new(5);
     /// ```
     #[cfg(not(no_global_oom_handling))]
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "rust1", since = "1.0.0")]
     #[must_use]
     #[rustc_diagnostic_item = "box_new"]
@@ -316,7 +316,7 @@ impl<T> Box<T> {
     #[cfg(not(no_global_oom_handling))]
     #[stable(feature = "pin", since = "1.33.0")]
     #[must_use]
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     pub fn pin(x: T) -> Pin<Box<T>> {
         Box::new(x).into()
     }
@@ -609,7 +609,7 @@ impl<T, A: Allocator> Box<T, A> {
     #[cfg(not(no_global_oom_handling))]
     #[unstable(feature = "allocator_api", issue = "32838")]
     #[must_use]
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     pub fn pin_in(x: T, alloc: A) -> Pin<Self>
     where
         A: 'static + Allocator,

diff --git a/library/alloc/src/collections/binary_heap/mod.rs b/library/alloc/src/collections/binary_heap/mod.rs
@@ -801,7 +801,7 @@ impl<T: Ord, A: Allocator> BinaryHeap<T, A> {
 
         let tail_len = self.len() - start;
 
-        #[inline(always)]
+        #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
         fn log2_fast(x: usize) -> usize {
             (usize::BITS - x.leading_zeros() - 1) as usize
         }

diff --git a/library/alloc/src/collections/linked_list.rs b/library/alloc/src/collections/linked_list.rs
@@ -1497,7 +1497,7 @@ impl<'a, T, A: Allocator> Cursor<'a, T, A> {
 
     /// Provides a reference to the cursor's parent list.
     #[must_use]
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     #[unstable(feature = "linked_list_cursors", issue = "58533")]
     pub fn as_list(&self) -> &'a LinkedList<T, A> {
         self.list
@@ -1619,7 +1619,7 @@ impl<'a, T, A: Allocator> CursorMut<'a, T, A> {
     /// `CursorMut`, which means it cannot outlive the `CursorMut` and that the
     /// `CursorMut` is frozen for the lifetime of the reference.
     #[must_use]
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     #[unstable(feature = "linked_list_cursors", issue = "58533")]
     pub fn as_list(&self) -> &LinkedList<T, A> {
         self.list

diff --git a/library/alloc/src/ffi/c_str.rs b/library/alloc/src/ffi/c_str.rs
@@ -273,7 +273,7 @@ impl CString {
         }
 
         // Specialization for avoiding reallocation
-        #[inline(always)] // Without that it is not inlined into specializations
+        #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] // Without that it is not inlined into specializations
         fn spec_new_impl_bytes(bytes: &[u8]) -> Result<CString, NulError> {
             // We cannot have such large slice that we would overflow here
             // but using `checked_add` allows LLVM to assume that capacity never overflows

diff --git a/library/alloc/src/rc.rs b/library/alloc/src/rc.rs
@@ -354,7 +354,7 @@ impl<T: ?Sized> Rc<T> {
 }
 
 impl<T: ?Sized, A: Allocator> Rc<T, A> {
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn inner(&self) -> &RcBox<T> {
         // This unsafety is ok because while this Rc is alive we're guaranteed
         // that the inner pointer is valid.
@@ -2207,7 +2207,7 @@ impl<T: Copy> RcFromSlice<T> for Rc<[T]> {
 impl<T: ?Sized, A: Allocator> Deref for Rc<T, A> {
     type Target = T;
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn deref(&self) -> &T {
         &self.inner().value
     }
@@ -2453,7 +2453,7 @@ impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for Rc<T, A> {
     ///
     /// assert_eq!(Some(Ordering::Less), five.partial_cmp(&Rc::new(6)));
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn partial_cmp(&self, other: &Rc<T, A>) -> Option<Ordering> {
         (**self).partial_cmp(&**other)
     }
@@ -2471,7 +2471,7 @@ impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for Rc<T, A> {
     ///
     /// assert!(five < Rc::new(6));
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn lt(&self, other: &Rc<T, A>) -> bool {
         **self < **other
     }
@@ -2489,7 +2489,7 @@ impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for Rc<T, A> {
     ///
     /// assert!(five <= Rc::new(5));
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn le(&self, other: &Rc<T, A>) -> bool {
         **self <= **other
     }
@@ -2507,7 +2507,7 @@ impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for Rc<T, A> {
     ///
     /// assert!(five > Rc::new(4));
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn gt(&self, other: &Rc<T, A>) -> bool {
         **self > **other
     }
@@ -2525,7 +2525,7 @@ impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for Rc<T, A> {
     ///
     /// assert!(five >= Rc::new(5));
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn ge(&self, other: &Rc<T, A>) -> bool {
         **self >= **other
     }
@@ -3527,24 +3527,24 @@ trait RcInnerPtr {
 }
 
 impl<T: ?Sized> RcInnerPtr for RcBox<T> {
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn weak_ref(&self) -> &Cell<usize> {
         &self.weak
     }
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn strong_ref(&self) -> &Cell<usize> {
         &self.strong
     }
 }
 
 impl<'a> RcInnerPtr for WeakInner<'a> {
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn weak_ref(&self) -> &Cell<usize> {
         self.weak
     }
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn strong_ref(&self) -> &Cell<usize> {
         self.strong
     }

diff --git a/library/alloc/src/task.rs b/library/alloc/src/task.rs
@@ -133,18 +133,18 @@ impl<W: Wake + Send + Sync + 'static> From<Arc<W>> for RawWaker {
 // trait dispatch - instead both impls call this function directly and
 // explicitly.
 #[cfg(target_has_atomic = "ptr")]
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
 fn raw_waker<W: Wake + Send + Sync + 'static>(waker: Arc<W>) -> RawWaker {
     // Increment the reference count of the arc to clone it.
     //
-    // The #[inline(always)] is to ensure that raw_waker and clone_waker are
+    // The #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] is to ensure that raw_waker and clone_waker are
     // always generated in the same code generation unit as one another, and
     // therefore that the structurally identical const-promoted RawWakerVTable
     // within both functions is deduplicated at LLVM IR code generation time.
     // This allows optimizing Waker::will_wake to a single pointer comparison of
     // the vtable pointers, rather than comparing all four function pointers
     // within the vtables.
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     unsafe fn clone_waker<W: Wake + Send + Sync + 'static>(waker: *const ()) -> RawWaker {
         unsafe { Arc::increment_strong_count(waker as *const W) };
         RawWaker::new(
@@ -310,13 +310,13 @@ impl<W: LocalWake + 'static> From<Rc<W>> for RawWaker {
 // the safety of `From<Rc<W>> for Waker` does not depend on the correct
 // trait dispatch - instead both impls call this function directly and
 // explicitly.
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
 fn local_raw_waker<W: LocalWake + 'static>(waker: Rc<W>) -> RawWaker {
     // Increment the reference count of the Rc to clone it.
     //
     // Refer to the comment on raw_waker's clone_waker regarding why this is
     // always inline.
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     unsafe fn clone_waker<W: LocalWake + 'static>(waker: *const ()) -> RawWaker {
         unsafe { Rc::increment_strong_count(waker as *const W) };
         RawWaker::new(

diff --git a/library/core/src/alloc/layout.rs b/library/core/src/alloc/layout.rs
@@ -84,7 +84,7 @@ impl Layout {
         true
     }
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     const fn max_size_for_align(align: Alignment) -> usize {
         // (power-of-two implies align != 0.)
 

diff --git a/library/core/src/alloc/mod.rs b/library/core/src/alloc/mod.rs
@@ -360,7 +360,7 @@ pub unsafe trait Allocator {
     /// Creates a "by reference" adapter for this instance of `Allocator`.
     ///
     /// The returned adapter also implements `Allocator` and will simply borrow this.
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     fn by_ref(&self) -> &Self
     where
         Self: Sized,

diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs
@@ -813,12 +813,12 @@ fn panic_already_mutably_borrowed(err: BorrowError) -> ! {
 type BorrowFlag = isize;
 const UNUSED: BorrowFlag = 0;
 
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
 fn is_writing(x: BorrowFlag) -> bool {
     x < UNUSED
 }
 
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
 fn is_reading(x: BorrowFlag) -> bool {
     x > UNUSED
 }
@@ -2079,7 +2079,7 @@ impl<T> UnsafeCell<T> {
     /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
     #[rustc_const_stable(feature = "const_unsafe_cell_new", since = "1.32.0")]
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     pub const fn new(value: T) -> UnsafeCell<T> {
         UnsafeCell { value }
     }
@@ -2095,7 +2095,7 @@ impl<T> UnsafeCell<T> {
     ///
     /// let five = uc.into_inner();
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "rust1", since = "1.0.0")]
     // When this is const stabilized, please remove `primitive_into_inner` below.
     #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")]
@@ -2119,7 +2119,7 @@ impl<T: ?Sized> UnsafeCell<T> {
     /// *uc.get_mut() -= 1;
     /// assert_eq!(*uc.get_mut(), 41);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     #[unstable(feature = "unsafe_cell_from_mut", issue = "111645")]
     pub const fn from_mut(value: &mut T) -> &mut UnsafeCell<T> {
         // SAFETY: `UnsafeCell<T>` has the same memory layout as `T` due to #[repr(transparent)].
@@ -2142,7 +2142,7 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// let five = uc.get();
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "rust1", since = "1.0.0")]
     #[rustc_const_stable(feature = "const_unsafecell_get", since = "1.32.0")]
     #[rustc_never_returns_null_ptr]
@@ -2168,7 +2168,7 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// assert_eq!(*c.get_mut(), 6);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "unsafe_cell_get_mut", since = "1.50.0")]
     #[rustc_const_unstable(feature = "const_unsafecell_get_mut", issue = "88836")]
     pub const fn get_mut(&mut self) -> &mut T {
@@ -2203,7 +2203,7 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// assert_eq!(uc.into_inner(), 5);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
     #[rustc_const_stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
     #[rustc_diagnostic_item = "unsafe_cell_raw_get"]

diff --git a/library/core/src/clone.rs b/library/core/src/clone.rs
@@ -321,7 +321,7 @@ mod impls {
             $(
                 #[stable(feature = "rust1", since = "1.0.0")]
                 impl Clone for $t {
-                    #[inline(always)]
+                    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                     fn clone(&self) -> Self {
                         *self
                     }
@@ -347,15 +347,15 @@ mod impls {
 
     #[stable(feature = "rust1", since = "1.0.0")]
     impl<T: ?Sized> Clone for *const T {
-        #[inline(always)]
+        #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
         fn clone(&self) -> Self {
             *self
         }
     }
 
     #[stable(feature = "rust1", since = "1.0.0")]
     impl<T: ?Sized> Clone for *mut T {
-        #[inline(always)]
+        #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
         fn clone(&self) -> Self {
             *self
         }
@@ -364,7 +364,7 @@ mod impls {
     /// Shared references can be cloned, but mutable references *cannot*!
     #[stable(feature = "rust1", since = "1.0.0")]
     impl<T: ?Sized> Clone for &T {
-        #[inline(always)]
+        #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
         #[rustc_diagnostic_item = "noop_method_clone"]
         fn clone(&self) -> Self {
             *self
-Original file line number
+Diff line change
@@ Expand Up / @@ -46,6 +46,7 @@ pub enum InlineAttr { @@
         Hint,
         Always,
         Never,
+        Usually,
     }
     #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]
@@ Expand Down @@