Update to nightly-2024-03-01

Author: wenyuzhao

buddy/src/lib.rs

@@ -1,18 +1,15 @@
 #![allow(incomplete_features)]
 #![feature(type_alias_impl_trait)]
 #![feature(thread_local)]
-#![feature(core_intrinsics)]
 
 extern crate mallockit;
 
-use core::alloc::Layout;
 use mallockit::{
     space::*,
     space::{freelist_space::*, large_object_space::*},
     util::*,
     Mutator, Plan,
 };
-use std::intrinsics::likely;
 
 const FREELIST_SPACE: SpaceId = SpaceId::DEFAULT;
 const LARGE_OBJECT_SPACE: SpaceId = SpaceId::LARGE_OBJECT_SPACE;
@@ -35,7 +32,7 @@ impl Plan for Buddy {
 
     fn get_layout(ptr: Address) -> Layout {
         debug_assert!(FREELIST_SPACE.contains(ptr) || LARGE_OBJECT_SPACE.contains(ptr));
-        if likely(FREELIST_SPACE.contains(ptr)) {
+        if FREELIST_SPACE.contains(ptr) {
             FreeListSpace::get_layout(ptr)
         } else {
             Self::get().large_object_space.get_layout::<Size4K>(ptr)
@@ -66,7 +63,7 @@ impl Mutator for BuddyMutator {
     const NEW: Self = Self::new();
 
     fn alloc(&mut self, layout: Layout) -> Option<Address> {
-        if likely(FreeListSpace::can_allocate(layout)) {
+        if FreeListSpace::can_allocate(layout) {
             mallockit::stat::track_allocation(layout, false);
             self.freelist.alloc(layout)
         } else {
@@ -77,7 +74,7 @@ impl Mutator for BuddyMutator {
 
     fn dealloc(&mut self, ptr: Address) {
         debug_assert!(FREELIST_SPACE.contains(ptr) || LARGE_OBJECT_SPACE.contains(ptr));
-        if likely(FREELIST_SPACE.contains(ptr)) {
+        if FREELIST_SPACE.contains(ptr) {
             mallockit::stat::track_deallocation(false);
             self.freelist.dealloc(ptr)
         } else {

bump/src/lib.rs

@@ -4,7 +4,6 @@
 
 extern crate mallockit;
 
-use core::alloc::Layout;
 use mallockit::{space::immortal_space::*, space::*, util::*, Mutator, Plan};
 
 const IMMORTAL_SPACE: SpaceId = SpaceId::DEFAULT;

hoard/src/hoard_space.rs

@@ -1,5 +1,3 @@
-use std::intrinsics::likely;
-
 use super::{page_resource::BlockPageResource, Allocator, Space, SpaceId};
 use crate::{pool::Pool, super_block::SuperBlock};
 use mallockit::util::{discrete_tlab::DiscreteTLAB, size_class::SizeClass, *};
@@ -103,7 +101,7 @@ impl HoardAllocator {
 impl Allocator for HoardAllocator {
     fn alloc(&mut self, layout: Layout) -> Option<Address> {
         let size_class = SizeClass::from_layout(layout);
-        if likely(layout.size() <= Self::LARGEST_SMALL_OBJECT) {
+        if layout.size() <= Self::LARGEST_SMALL_OBJECT {
             if let Some(cell) = self.tlab.pop(size_class) {
                 return Some(cell);
             }
@@ -114,10 +112,9 @@ impl Allocator for HoardAllocator {
     fn dealloc(&mut self, cell: Address) {
         let block = SuperBlock::containing(cell);
         let size = block.size_class.bytes();
-        if likely(
-            size <= Self::LARGEST_SMALL_OBJECT
-                && size + self.tlab.free_bytes() <= Self::LOCAL_HEAP_THRESHOLD,
-        ) {
+        if size <= Self::LARGEST_SMALL_OBJECT
+            && size + self.tlab.free_bytes() <= Self::LOCAL_HEAP_THRESHOLD
+        {
             self.tlab.push(block.size_class, cell);
         } else {
             self.local.free_cell(cell, &self.space);

hoard/src/lib.rs

@@ -1,10 +1,8 @@
 #![allow(incomplete_features)]
 #![feature(thread_local)]
-#![feature(core_intrinsics)]
 #![feature(const_mut_refs)]
 #![feature(const_trait_impl)]
 #![feature(step_trait)]
-#![feature(const_likely)]
 
 #[allow(unused)]
 #[macro_use]
@@ -14,10 +12,8 @@ mod hoard_space;
 mod pool;
 mod super_block;
 
-use core::alloc::Layout;
 use hoard_space::*;
 use mallockit::{space::large_object_space::*, space::*, util::*, Mutator, Plan};
-use std::intrinsics::likely;
 
 const HOARD_SPACE: SpaceId = SpaceId::DEFAULT;
 const LARGE_OBJECT_SPACE: SpaceId = SpaceId::LARGE_OBJECT_SPACE;
@@ -40,7 +36,7 @@ impl Plan for Hoard {
 
     fn get_layout(ptr: Address) -> Layout {
         debug_assert!(HOARD_SPACE.contains(ptr) || LARGE_OBJECT_SPACE.contains(ptr));
-        if likely(HOARD_SPACE.contains(ptr)) {
+        if HOARD_SPACE.contains(ptr) {
             HoardSpace::get_layout(ptr)
         } else {
             Self::get().large_object_space.get_layout::<Size4K>(ptr)
@@ -68,7 +64,7 @@ impl Mutator for HoardMutator {
     const NEW: Self = Self::new();
 
     fn alloc(&mut self, layout: Layout) -> Option<Address> {
-        if likely(HoardSpace::can_allocate(layout)) {
+        if HoardSpace::can_allocate(layout) {
             mallockit::stat::track_allocation(layout, false);
             self.hoard.alloc(layout)
         } else {
@@ -79,7 +75,7 @@ impl Mutator for HoardMutator {
 
     fn dealloc(&mut self, ptr: Address) {
         debug_assert!(HOARD_SPACE.contains(ptr) || LARGE_OBJECT_SPACE.contains(ptr));
-        if likely(HOARD_SPACE.contains(ptr)) {
+        if HOARD_SPACE.contains(ptr) {
             mallockit::stat::track_deallocation(false);
             self.hoard.dealloc(ptr)
         } else {

hoard/src/pool.rs

@@ -2,10 +2,7 @@ use crate::{hoard_space::HoardSpace, super_block::SuperBlock};
 use array_const_fn_init::array_const_fn_init;
 use mallockit::util::{size_class::SizeClass, Address, Lazy, Local};
 use spin::{relax::Yield, MutexGuard};
-use std::{
-    intrinsics::likely,
-    sync::atomic::{AtomicUsize, Ordering},
-};
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 type Mutex<T> = spin::mutex::Mutex<T, Yield>;
 
@@ -134,12 +131,12 @@ impl BlockList {
 
     #[cold]
     fn move_to_front_slow(&mut self, mut block: SuperBlock, alloc: bool) {
-        if likely(Some(block) == self.cache) {
+        if Some(block) == self.cache {
             return;
         }
         let group = Self::group(block, alloc);
         let block_group = block.group as usize;
-        if likely(Some(block) == self.groups[group] || group == block_group) {
+        if Some(block) == self.groups[group] || group == block_group {
             return;
         }
         if self.groups[block_group] == Some(block) {
@@ -161,7 +158,7 @@ impl BlockList {
     }
 
     fn move_to_front(&mut self, block: SuperBlock, alloc: bool) {
-        if likely(Some(block) == self.cache) {
+        if Some(block) == self.cache {
             return;
         }
         self.move_to_front_slow(block, alloc)

hoard/src/super_block.rs

@@ -1,7 +1,4 @@
-use std::{
-    intrinsics::{likely, unlikely},
-    num::NonZeroUsize,
-};
+use std::num::NonZeroUsize;
 
 use mallockit::util::{aligned_block::AlignedBlockConfig, size_class::SizeClass};
 
@@ -64,17 +61,17 @@ impl SuperBlock {
     }
 
     pub fn alloc_cell(mut self) -> Option<Address> {
-        let cell = if unlikely(self.head_cell.is_zero()) {
-            if likely(self.bump_cursor < Self::BYTES as u32) {
+        let cell = if !self.head_cell.is_zero() {
+            self.head_cell
+        } else {
+            if self.bump_cursor < Self::BYTES as u32 {
                 let cell = self.start() + (self.bump_cursor as usize);
                 self.bump_cursor = self.bump_cursor + self.size_class.bytes() as u32;
                 self.used_bytes += self.size_class.bytes() as u32;
                 return Some(cell);
             } else {
                 return None;
             }
-        } else {
-            self.head_cell
         };
         self.head_cell = unsafe { cell.load::<Address>() };
         self.used_bytes += self.size_class.bytes() as u32;

mallockit/src/lib.rs

@@ -1,11 +1,11 @@
 #![allow(incomplete_features)]
+#![allow(internal_features)]
 #![feature(core_intrinsics)]
 #![feature(const_trait_impl)]
 #![feature(const_mut_refs)]
 #![feature(const_ptr_is_null)]
 #![feature(type_ascription)]
 #![feature(step_trait)]
-#![feature(const_likely)]
 #![feature(thread_local)]
 #![feature(allocator_api)]
 #![feature(never_type)]
@@ -21,6 +21,7 @@
 #![feature(type_alias_impl_trait)]
 #![feature(specialization)]
 #![feature(const_for)]
+#![feature(effects)]
 
 extern crate mallockit_proc_macro;
 

mallockit/src/log.rs

@@ -1,8 +1,6 @@
-use libc;
 use spin::Mutex;
 use std::fmt;
 use std::fmt::Write;
-use std::intrinsics::unlikely;
 
 #[doc(hidden)]
 #[cold]
@@ -76,7 +74,7 @@ impl Log {
     fn put_char(&mut self, c: u8) {
         self.buffer[self.cursor] = c;
         self.cursor += 1;
-        if unlikely(self.cursor >= self.buffer.len()) {
+        if self.cursor >= self.buffer.len() {
             self.flush();
         }
     }

mallockit/src/malloc.rs

@@ -4,13 +4,13 @@ use crate::util::Address;
 use crate::util::Lazy;
 use crate::Mutator;
 use core::{alloc::Layout, ptr};
-use std::intrinsics::unlikely;
+use std::marker::PhantomData;
 
 pub trait GetMutatorType {
     type Mutator: Mutator;
 }
 
-pub struct MallocAPI<P: Plan>(&'static Lazy<P>);
+pub struct MallocAPI<P: Plan>(PhantomData<P>);
 
 impl<P: Plan> GetMutatorType for MallocAPI<P> {
     type Mutator = P::Mutator;
@@ -31,7 +31,7 @@ impl<P: Plan> MallocAPI<P> {
     pub const PAGE_SIZE: usize = 4096;
 
     pub const fn new(plan: &'static Lazy<P>) -> Self {
-        Self(plan)
+        Self(PhantomData)
     }
 
     pub const fn new_mutator() -> P::Mutator {
@@ -58,7 +58,7 @@ impl<P: Plan> MallocAPI<P> {
     pub unsafe fn malloc_size(&self, ptr: Address) -> usize {
         let ptr = Address::from(ptr);
         #[cfg(target_os = "macos")]
-        if unlikely(Self::zero_spaceid(ptr.into())) {
+        if Self::zero_spaceid(ptr.into()) {
             return crate::util::macos_malloc_zone::external_memory_size(ptr);
         }
         P::get_layout(ptr).size()
@@ -85,11 +85,11 @@ impl<P: Plan> MallocAPI<P> {
     }
 
     pub unsafe fn free(&self, ptr: *mut u8) {
-        if unlikely(ptr.is_null()) {
+        if ptr.is_null() {
             return;
         }
         #[cfg(target_os = "macos")]
-        if unlikely(Self::zero_spaceid(ptr.into())) {
+        if Self::zero_spaceid(ptr.into()) {
             return;
         }
         self.mutator().dealloc(ptr.into());
@@ -105,14 +105,14 @@ impl<P: Plan> MallocAPI<P> {
         if ptr.is_null() {
             return self.alloc_or_enomem(new_size, Self::MIN_ALIGNMENT);
         }
-        if unlikely(free_if_new_size_is_zero && new_size == 0) {
+        if free_if_new_size_is_zero && new_size == 0 {
             self.free(ptr);
             return ptr::null_mut();
         }
         let new_size = Self::align_up(new_size, Self::MIN_ALIGNMENT);
 
         #[cfg(target_os = "macos")]
-        if unlikely(Self::zero_spaceid(ptr.into())) {
+        if Self::zero_spaceid(ptr.into()) {
             let ptr = Address::from(ptr);
             let old_size = crate::util::macos_malloc_zone::external_memory_size(ptr);
             let new_layout =
@@ -152,7 +152,7 @@ impl<P: Plan> MallocAPI<P> {
         mut alignment: usize,
         size: usize,
     ) -> i32 {
-        if unlikely(!alignment.is_power_of_two()) {
+        if !alignment.is_power_of_two() {
             return libc::EINVAL;
         }
         alignment = std::cmp::max(alignment, Self::MIN_ALIGNMENT);
@@ -168,7 +168,7 @@ impl<P: Plan> MallocAPI<P> {
     pub unsafe fn memalign(&self, alignment: usize, size: usize) -> *mut u8 {
         let mut result = ptr::null_mut();
         let errno = self.posix_memalign(&mut result, alignment, size);
-        if unlikely(result.is_null()) {
+        if result.is_null() {
             Self::set_error(errno)
         }
         result
@@ -181,11 +181,10 @@ impl<P: Plan> MallocAPI<P> {
         einval_if_size_is_not_aligned: bool,
         einval_if_size_is_zero: bool,
     ) -> *mut u8 {
-        if unlikely(
-            !alignment.is_power_of_two()
-                || (einval_if_size_is_not_aligned && (size & (alignment - 1)) != 0)
-                || (einval_if_size_is_zero && size == 0),
-        ) {
+        if !alignment.is_power_of_two()
+            || (einval_if_size_is_not_aligned && (size & (alignment - 1)) != 0)
+            || (einval_if_size_is_zero && size == 0)
+        {
             Self::set_error(libc::EINVAL);
             return ptr::null_mut();
         }

mallockit/src/mutator.rs

@@ -1,5 +1,4 @@
 use std::alloc::Layout;
-use std::intrinsics::unlikely;
 use std::ptr;
 
 use crate::plan::Plan;
@@ -33,7 +32,7 @@ pub trait Mutator: Sized + 'static + TLS {
 
     fn realloc(&mut self, ptr: Address, new_size: usize) -> Option<Address> {
         let layout = Self::Plan::get_layout(ptr);
-        if unlikely(layout.size() >= new_size) {
+        if layout.size() >= new_size {
             return Some(ptr);
         }
         let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
@@ -64,7 +63,7 @@ impl InternalTLS {
 
     #[cfg(not(target_os = "macos"))]
     pub fn current() -> &'static mut Self {
-        unsafe { &mut INTERNAL_TLS }
+        unsafe { &mut *INTERNAL_TLS.get() }
     }
 
     #[cfg(target_os = "macos")]
@@ -76,7 +75,8 @@ impl InternalTLS {
 
 #[cfg(not(target_os = "macos"))]
 #[thread_local]
-static mut INTERNAL_TLS: InternalTLS = InternalTLS::NEW;
+static mut INTERNAL_TLS: std::cell::UnsafeCell<InternalTLS> =
+    std::cell::UnsafeCell::new(InternalTLS::NEW);
 
 pub trait TLS: Sized {
     const NEW: Self;
@@ -127,7 +127,7 @@ mod macos_tls {
     #[allow(unused)]
     pub(super) fn get_internal_tls() -> *mut InternalTLS {
         let mut tls = _get_tls::<InternalTLS>();
-        if unlikely(tls.is_null()) {
+        if tls.is_null() {
             unsafe {
                 mallockit_initialize_macos_tls();
             }
@@ -140,7 +140,7 @@ mod macos_tls {
     #[allow(unused)]
     pub(super) fn get_tls<T: TLS>() -> *mut T {
         let mut tls = _get_tls::<(InternalTLS, T)>();
-        if unlikely(tls.is_null()) {
+        if tls.is_null() {
             tls = init_tls::<T>();
         }
         unsafe { &mut (*tls).1 }