Skip to content

Commit 9da53a0

Browse files
wenyuzhaowenyuzhao
authored
Dynamic Heap Virtual Address Space (#4)
* wip * Pass tests * Cleanup * Fix macos aarch64 errors
1 parent 6c780b0 commit 9da53a0

13 files changed

+121
-81
lines changed

‎buddy/src/lib.rs

+3-4
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,9 @@ struct BuddyMutator {
4747
impl BuddyMutator {
4848
const fn new() -> Self {
4949
Self {
50-
freelist: FreeListAllocator::new(
51-
Lazy::new(|| &Self::plan().freelist_space),
52-
FREELIST_SPACE,
53-
),
50+
freelist: FreeListAllocator::new::<FREELIST_SPACE>(Lazy::new(|| {
51+
&Self::plan().freelist_space
52+
})),
5453
los: LargeObjectAllocator::new(Lazy::new(|| &Self::plan().large_object_space)),
5554
}
5655
}

‎mallockit/src/lib.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@
1010
#![feature(adt_const_params)]
1111
#![feature(generic_const_exprs)]
1212
#![feature(effects)]
13+
#![feature(asm_const)]
1314

1415
extern crate mallockit_proc_macro;
16+
pub extern crate spin;
1517

1618
#[macro_use]
1719
pub mod log;
@@ -39,8 +41,9 @@ const ERROR: ! = "32-bit is not supported";
3941

4042
#[cfg(not(any(
4143
all(target_os = "linux", target_arch = "x86_64"),
42-
all(target_os = "macos", target_arch = "x86_64"),
4344
all(target_os = "linux", target_arch = "aarch64"),
45+
all(target_os = "macos", target_arch = "x86_64"),
46+
all(target_os = "macos", target_arch = "aarch64"),
4447
)))]
4548
const ERROR: ! = r#"
4649
❌ Unsupported Platform.

‎mallockit/src/malloc.rs

+6-10
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use super::Plan;
2-
use crate::space::SpaceId;
2+
use crate::util::heap::HEAP;
33
use crate::util::Address;
44
use crate::util::Lazy;
55
use crate::Mutator;
@@ -34,16 +34,12 @@ impl<P: Plan> MallocAPI<P> {
3434
Self(PhantomData)
3535
}
3636

37-
pub const fn new_mutator() -> P::Mutator {
38-
P::Mutator::NEW
39-
}
40-
4137
pub fn mutator(&self) -> &'static mut P::Mutator {
4238
P::Mutator::current()
4339
}
4440

45-
pub fn zero_spaceid(a: Address) -> bool {
46-
SpaceId::from(a).is_invalid()
41+
pub fn is_in_mallockit_heap(a: Address) -> bool {
42+
HEAP.contains(a)
4743
}
4844

4945
pub const fn align_up(value: usize, align: usize) -> usize {
@@ -58,7 +54,7 @@ impl<P: Plan> MallocAPI<P> {
5854
pub unsafe fn malloc_size(&self, ptr: Address) -> usize {
5955
let ptr = Address::from(ptr);
6056
#[cfg(target_os = "macos")]
61-
if Self::zero_spaceid(ptr.into()) {
57+
if Self::is_in_mallockit_heap(ptr.into()) {
6258
return crate::util::macos_malloc_zone::external_memory_size(ptr);
6359
}
6460
P::get_layout(ptr).size()
@@ -89,7 +85,7 @@ impl<P: Plan> MallocAPI<P> {
8985
return;
9086
}
9187
#[cfg(target_os = "macos")]
92-
if Self::zero_spaceid(ptr.into()) {
88+
if Self::is_in_mallockit_heap(ptr.into()) {
9389
return;
9490
}
9591
self.mutator().dealloc(ptr.into());
@@ -112,7 +108,7 @@ impl<P: Plan> MallocAPI<P> {
112108
let new_size = Self::align_up(new_size, Self::MIN_ALIGNMENT);
113109

114110
#[cfg(target_os = "macos")]
115-
if Self::zero_spaceid(ptr.into()) {
111+
if Self::is_in_mallockit_heap(ptr.into()) {
116112
let ptr = Address::from(ptr);
117113
let old_size = crate::util::macos_malloc_zone::external_memory_size(ptr);
118114
let new_layout =

‎mallockit/src/mutator.rs

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use crate::util::Address;
77

88
pub trait Mutator: Sized + 'static + TLS {
99
type Plan: Plan<Mutator = Self>;
10+
1011
const NEW: Self;
1112

1213
fn current() -> &'static mut Self {

‎mallockit/src/space/freelist_space.rs

+6-3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ use super::{page_resource::BlockPageResource, Allocator, Space, SpaceId};
22
use crate::util::bits::{BitField, BitFieldSlot};
33
use crate::util::freelist::intrusive_freelist::AddressSpaceConfig;
44
use crate::util::freelist::intrusive_freelist::IntrusiveFreeList;
5+
use crate::util::heap::HEAP;
56
use crate::util::*;
67
use spin::Mutex;
78
use std::{ops::Range, sync::atomic::AtomicUsize};
@@ -122,14 +123,16 @@ impl Cell {
122123

123124
pub struct FreeListAllocator {
124125
space: Lazy<&'static FreeListSpace, Local>,
125-
freelist: IntrusiveFreeList<AddressSpace>,
126+
freelist: Lazy<IntrusiveFreeList<AddressSpace>, Local>,
126127
}
127128

128129
impl FreeListAllocator {
129-
pub const fn new(space: Lazy<&'static FreeListSpace, Local>, space_id: SpaceId) -> Self {
130+
pub const fn new<const SPACE_ID: SpaceId>(space: Lazy<&'static FreeListSpace, Local>) -> Self {
130131
Self {
131132
space,
132-
freelist: IntrusiveFreeList::new(false, space_id.address_space().start),
133+
freelist: Lazy::new(|| {
134+
IntrusiveFreeList::new(false, HEAP.get_space_range(SPACE_ID).start)
135+
}),
133136
}
134137
}
135138

‎mallockit/src/space/mod.rs

+3-13
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@ pub mod large_object_space;
88
pub mod meta;
99
pub mod page_resource;
1010
pub(crate) mod page_table;
11+
use std::marker::ConstParamTy;
1112

1213
#[repr(transparent)]
13-
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14-
pub struct SpaceId(u8);
14+
#[derive(Debug, Clone, Copy, PartialEq, Eq, ConstParamTy)]
15+
pub struct SpaceId(pub(crate) u8);
1516

1617
impl SpaceId {
17-
pub(crate) const HEAP_START: Address = Address(1usize << 45);
1818
pub const LOG_MAX_SPACE_SIZE: usize = 41;
1919
pub(crate) const SHIFT: usize = Self::LOG_MAX_SPACE_SIZE;
2020
pub(crate) const MASK: usize = 0b1111 << Self::SHIFT;
@@ -32,10 +32,6 @@ impl SpaceId {
3232
}
3333
}
3434

35-
pub const fn is_invalid(&self) -> bool {
36-
self.0 == 0 || self.0 == 15
37-
}
38-
3935
pub fn from(addr: Address) -> Self {
4036
let id = (usize::from(addr) & Self::MASK) >> Self::SHIFT;
4137
Self(id as u8)
@@ -44,12 +40,6 @@ impl SpaceId {
4440
pub fn contains(&self, addr: Address) -> bool {
4541
Self::from(addr).0 == self.0
4642
}
47-
48-
pub const fn address_space(&self) -> Range<Address> {
49-
let start = Address(Self::HEAP_START.0 + ((self.0 as usize) << Self::LOG_MAX_SPACE_SIZE));
50-
let end = Address(start.0 + (1usize << Self::LOG_MAX_SPACE_SIZE));
51-
start..end
52-
}
5343
}
5444

5545
pub trait Space: Sized + 'static {

‎mallockit/src/space/page_resource/block_page_resource.rs

+4-22
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use super::super::SpaceId;
22
use super::PageResource;
3-
use crate::util::memory::RawMemory;
3+
use crate::util::heap::HEAP;
44
use crate::util::*;
55
use atomic::Atomic;
66
use crossbeam::queue::SegQueue;
@@ -23,32 +23,17 @@ impl BlockPageResource {
2323
pub fn new(id: SpaceId, log_bytes: usize) -> Self {
2424
debug_assert!(id.0 < 0b0000_1111);
2525
debug_assert!(log_bytes >= Size4K::LOG_BYTES);
26+
let range = HEAP.get_space_range(id);
2627
Self {
2728
id,
2829
log_bytes,
29-
cursor: Atomic::new(id.address_space().start),
30-
highwater: id.address_space().end,
30+
cursor: Atomic::new(range.start),
31+
highwater: range.end,
3132
recycled_blocks: SegQueue::new(),
3233
reserved_bytes: AtomicUsize::new(0),
3334
}
3435
}
3536

36-
fn map_pages<S: PageSize>(&self, start: Page<S>, pages: usize) -> bool {
37-
let size = pages << S::LOG_BYTES;
38-
match RawMemory::map(start.start(), size) {
39-
Ok(_) => {
40-
#[cfg(target_os = "linux")]
41-
if cfg!(feature = "transparent_huge_page") && S::LOG_BYTES != Size4K::LOG_BYTES {
42-
unsafe {
43-
libc::madvise(start.start().as_mut_ptr(), size, libc::MADV_HUGEPAGE);
44-
}
45-
}
46-
true
47-
}
48-
_ => false,
49-
}
50-
}
51-
5237
#[cold]
5338
fn acquire_block_slow<S: PageSize>(&self, pages: usize) -> Option<Range<Page<S>>> {
5439
debug_assert!(self.log_bytes >= S::LOG_BYTES);
@@ -65,9 +50,6 @@ impl BlockPageResource {
6550
match block {
6651
Ok(addr) => {
6752
let start = Page::<S>::new(addr);
68-
if !self.map_pages(start, pages) {
69-
return self.acquire_block_slow(pages); // Retry
70-
}
7153
let end = Step::forward(start, pages);
7254
return Some(start..end);
7355
}

‎mallockit/src/space/page_resource/freelist_page_resource.rs

+12-23
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use super::super::SpaceId;
22
use super::PageResource;
33
use crate::util::freelist::page_freelist::PageFreeList;
4+
use crate::util::heap::HEAP;
45
use crate::util::memory::RawMemory;
56
use crate::util::*;
67
use spin::mutex::Mutex;
@@ -20,49 +21,39 @@ pub struct FreelistPageResource {
2021
freelist: Mutex<PageFreeList<{ NUM_SIZE_CLASS }>, Yield>,
2122
reserved_bytes: AtomicUsize,
2223
meta: RwLock<Vec<AtomicU32>, Yield>,
24+
base: Address,
2325
}
2426

2527
impl FreelistPageResource {
2628
pub fn new(id: SpaceId) -> Self {
2729
debug_assert!(id.0 < 0b0000_1111);
28-
let base = id.address_space().start;
30+
let range = HEAP.get_space_range(id);
31+
let base = range.start;
2932
let mut freelist = PageFreeList::new(base);
3033
freelist.release_cell(base, 1 << (NUM_SIZE_CLASS - 1));
3134
Self {
3235
id,
3336
freelist: Mutex::new(freelist),
3437
reserved_bytes: AtomicUsize::new(0),
3538
meta: RwLock::new(unsafe { std::mem::transmute(vec![0u32; 1 << 20]) }),
39+
base,
3640
}
3741
}
3842

39-
fn map_pages<S: PageSize>(&self, start: Page<S>, pages: usize) -> bool {
40-
let size = pages << S::LOG_BYTES;
41-
match RawMemory::map(start.start(), size) {
42-
Ok(_) => {
43-
#[cfg(target_os = "linux")]
44-
if cfg!(feature = "transparent_huge_page") && S::LOG_BYTES != Size4K::LOG_BYTES {
45-
unsafe {
46-
libc::madvise(start.start().as_mut_ptr(), size, libc::MADV_HUGEPAGE);
47-
}
48-
}
49-
self.reserved_bytes
50-
.fetch_add(pages << S::LOG_BYTES, Ordering::SeqCst);
51-
true
52-
}
53-
_ => false,
54-
}
43+
fn map_pages<S: PageSize>(&self, _start: Page<S>, pages: usize) {
44+
self.reserved_bytes
45+
.fetch_add(pages << S::LOG_BYTES, Ordering::SeqCst);
5546
}
5647

5748
fn unmap_pages<S: PageSize>(&self, start: Page<S>, pages: usize) {
58-
RawMemory::unmap(start.start(), pages << S::LOG_BYTES);
49+
RawMemory::madv_free(start.start(), pages << S::LOG_BYTES);
5950
self.reserved_bytes
6051
.fetch_sub(pages << S::LOG_BYTES, Ordering::SeqCst);
6152
}
6253

6354
fn set_meta<S: PageSize>(&self, start: Page<S>, pages: usize) {
6455
debug_assert!(pages <= u32::MAX as usize);
65-
let index = (start.start() - self.id.address_space().start) >> Page::<Size4K>::LOG_BYTES;
56+
let index = (start.start() - self.base) >> Page::<Size4K>::LOG_BYTES;
6657
let meta = self.meta.upgradeable_read();
6758
if index >= meta.len() {
6859
let mut meta = meta.upgrade();
@@ -75,7 +66,7 @@ impl FreelistPageResource {
7566
}
7667

7768
fn get_meta<S: PageSize>(&self, start: Page<S>) -> usize {
78-
let index = (start.start() - self.id.address_space().start) >> Page::<Size4K>::LOG_BYTES;
69+
let index = (start.start() - self.base) >> Page::<Size4K>::LOG_BYTES;
7970
self.meta.read()[index].load(Ordering::Relaxed) as _
8071
}
8172
}
@@ -90,9 +81,7 @@ impl PageResource for FreelistPageResource {
9081
let units = pages << (S::LOG_BYTES - Size4K::LOG_BYTES);
9182
let start = self.freelist.lock().allocate_cell(units)?.start;
9283
let start = Page::<S>::new(start);
93-
if !self.map_pages(start, pages) {
94-
return self.acquire_pages(pages); // Retry
95-
}
84+
self.map_pages(start, pages);
9685
let end = Step::forward(start, pages);
9786
self.set_meta(start, units);
9887
Some(start..end)

‎mallockit/src/util/heap.rs

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
use std::ops::Range;
2+
3+
use spin::Lazy;
4+
5+
use crate::space::SpaceId;
6+
7+
use super::{memory::RawMemory, Address};
8+
9+
const LOG_HEAP_SIZE: usize = 45;
10+
const HEAP_SIZE: usize = 1 << LOG_HEAP_SIZE;
11+
12+
pub static HEAP: Lazy<Heap> = Lazy::new(|| Heap::new());
13+
14+
pub struct Heap {
15+
pub(crate) start: Address,
16+
pub(crate) end: Address,
17+
}
18+
19+
impl Heap {
20+
fn new() -> Self {
21+
let start = RawMemory::map_heap(HEAP_SIZE).unwrap();
22+
let end = start + HEAP_SIZE;
23+
Self { start, end }
24+
}
25+
26+
pub fn contains(&self, ptr: Address) -> bool {
27+
self.start <= ptr && ptr < self.end
28+
}
29+
30+
pub fn start(&self) -> Address {
31+
self.start
32+
}
33+
34+
pub fn end(&self) -> Address {
35+
self.end
36+
}
37+
38+
pub fn get_space_range(&self, id: SpaceId) -> Range<Address> {
39+
let start = self.start + ((id.0 as usize) << SpaceId::LOG_MAX_SPACE_SIZE);
40+
let end = start + (1usize << SpaceId::LOG_MAX_SPACE_SIZE);
41+
start..end
42+
}
43+
}

‎mallockit/src/util/lazy.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ impl ThreadLocality for Shared {
2929
}
3030
}
3131

32-
pub struct Lazy<T, TL: ThreadLocality = Shared, F: FnOnce() -> T = fn() -> T> {
32+
pub struct Lazy<T, TL: ThreadLocality = Shared, F = fn() -> T> {
3333
state: AtomicU8,
3434
value: UnsafeCell<MaybeUninit<T>>,
3535
init: Cell<Option<F>>,

‎mallockit/src/util/macos_malloc_zone.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use std::ptr::addr_of;
2+
13
use super::Address;
24

35
const ZONE_NAME: &'static [i8] = &[
@@ -18,7 +20,7 @@ pub static mut MALLOCKIT_MALLOC_ZONE: MallocZone = MallocZone {
1820
batch_malloc: 0 as _,
1921
// Nullable
2022
batch_free: 0 as _,
21-
introspect: unsafe { &MALLOCKIT_MALLOC_INTROSPECTION },
23+
introspect: unsafe { addr_of!(MALLOCKIT_MALLOC_INTROSPECTION) },
2224
version: 9,
2325
memalign: zone_memalign,
2426
free_definite_size: 0 as _,

0 commit comments

Comments
 (0)