Refactor Metadata (#342)

* remove gc_byte and header_byte, and move per-object metadata to VM object model
* restructure side_metadata to a submodule of metadata
* add header_metadata as a submodule of metadata
* suggest more metadata access inlining
* improve object forwading code style

Author: Javad Amiri

Cargo.toml

@@ -64,9 +64,6 @@ nogc_no_zeroing = ["nogc_lock_free"]
 # Q: Why do we need this as a compile time flat? We can always set the number of GC threads through options.
 single_worker = []
 
-# To support VMs that do not provide in-header GC byte
-side_gc_header = []
-
 # To run expensive comprehensive runtime checks, such as checking duplicate edges
 extreme_assertions = []
 

docs/tutorial/code/mygc_semispace/gc_work.rs

@@ -4,7 +4,7 @@ use crate::plan::CopyContext;
 use crate::policy::space::Space;
 use crate::scheduler::gc_work::*;
 use crate::util::alloc::{Allocator, BumpAllocator};
-use crate::util::forwarding_word;
+use crate::util::object_forwarding;
 use crate::util::{Address, ObjectReference};
 use crate::util::opaque_pointer::*;
 use crate::vm::VMBinding;
@@ -61,7 +61,7 @@ impl<VM: VMBinding> CopyContext for MyGCCopyContext<VM> {
         _bytes: usize,
         _semantics: crate::AllocationSemantics,
     ) {
-        forwarding_word::clear_forwarding_bits::<VM>(obj);
+        object_forwarding::clear_forwarding_bits::<VM>(obj);
     }
     // ANCHOR_END: copycontext_post_copy
 }

docs/tutorial/code/mygc_semispace/global.rs

@@ -18,7 +18,7 @@ use crate::util::heap::layout::heap_layout::VMMap;
 use crate::util::heap::layout::vm_layout_constants::{HEAP_END, HEAP_START};
 use crate::util::heap::HeapMeta;
 use crate::util::heap::VMRequest;
-use crate::util::side_metadata::{SideMetadataSanity, SideMetadataContext};
+use crate::util::metadata::side_metadata::{SideMetadataSanity, SideMetadataContext};
 use crate::util::options::UnsafeOptionsWrapper;
 use crate::util::opaque_pointer::*;
 use crate::vm::VMBinding;

src/memory_manager.rs

@@ -18,7 +18,7 @@ use crate::scheduler::GCWorker;
 use crate::scheduler::Work;
 use crate::scheduler::WorkBucketStage;
 use crate::util::alloc::allocators::AllocatorSelector;
-use crate::util::constants::LOG_BYTES_IN_PAGE;
+use crate::util::constants::{LOG_BYTES_IN_PAGE, MIN_OBJECT_SIZE};
 use crate::util::heap::layout::vm_layout_constants::HEAP_END;
 use crate::util::heap::layout::vm_layout_constants::HEAP_START;
 use crate::util::opaque_pointer::*;
@@ -108,8 +108,7 @@ pub fn alloc<VM: VMBinding>(
     // their object sizes are all larger than MMTk's min object size, so we simply put an assertion here.
     // If you plan to use MMTk with a VM with its object size smaller than MMTk's min object size, you should
     // meet the min object size in the fastpath.
-    #[cfg(debug_assertions)]
-    crate::util::forwarding_word::check_alloc_size::<VM>(size);
+    debug_assert!(size >= MIN_OBJECT_SIZE);
     mutator.alloc(size, align, offset, semantics)
 }
 

src/plan/barriers.rs

@@ -1,9 +1,12 @@
 //! Read/Write barrier implementations.
 
+use atomic::Ordering;
+
 use crate::scheduler::gc_work::*;
 use crate::scheduler::WorkBucketStage;
-use crate::util::side_metadata::*;
+use crate::util::metadata::{compare_exchange_metadata, MetadataSpec};
 use crate::util::*;
+use crate::vm::VMBinding;
 use crate::MMTK;
 
 /// BarrierSelector describes which barrier to use.
@@ -34,12 +37,12 @@ impl Barrier for NoBarrier {
 pub struct ObjectRememberingBarrier<E: ProcessEdgesWork> {
     mmtk: &'static MMTK<E::VM>,
     modbuf: Vec<ObjectReference>,
-    meta: SideMetadataSpec,
+    meta: MetadataSpec,
 }
 
 impl<E: ProcessEdgesWork> ObjectRememberingBarrier<E> {
     #[allow(unused)]
-    pub fn new(mmtk: &'static MMTK<E::VM>, meta: SideMetadataSpec) -> Self {
+    pub fn new(mmtk: &'static MMTK<E::VM>, meta: MetadataSpec) -> Self {
         Self {
             mmtk,
             modbuf: vec![],
@@ -48,8 +51,16 @@ impl<E: ProcessEdgesWork> ObjectRememberingBarrier<E> {
     }
 
     #[inline(always)]
-    fn enqueue_node(&mut self, obj: ObjectReference) {
-        if compare_exchange_atomic(self.meta, obj.to_address(), 0b1, 0b0) {
+    fn enqueue_node<VM: VMBinding>(&mut self, obj: ObjectReference) {
+        if compare_exchange_metadata::<VM>(
+            self.meta,
+            obj,
+            0b1,
+            0b0,
+            None,
+            Ordering::SeqCst,
+            Ordering::SeqCst,
+        ) {
             self.modbuf.push(obj);
             if self.modbuf.len() >= E::CAPACITY {
                 self.flush();
@@ -78,7 +89,7 @@ impl<E: ProcessEdgesWork> Barrier for ObjectRememberingBarrier<E> {
     fn post_write_barrier(&mut self, target: WriteTarget) {
         match target {
             WriteTarget::Object(obj) => {
-                self.enqueue_node(obj);
+                self.enqueue_node::<E::VM>(obj);
             }
             _ => unreachable!(),
         }

src/plan/gencopy/gc_work.rs

@@ -1,13 +1,15 @@
+use atomic::Ordering;
+
 use super::global::GenCopy;
 use crate::plan::PlanConstraints;
 use crate::plan::{barriers::BarrierSelector, CopyContext};
 use crate::policy::space::Space;
 use crate::scheduler::gc_work::*;
 use crate::scheduler::WorkerLocal;
 use crate::util::alloc::{Allocator, BumpAllocator};
-use crate::util::forwarding_word;
+use crate::util::metadata::store_metadata;
+use crate::util::object_forwarding;
 use crate::util::opaque_pointer::*;
-use crate::util::side_metadata::*;
 use crate::util::{Address, ObjectReference};
 use crate::vm::*;
 use crate::MMTK;
@@ -53,9 +55,15 @@ impl<VM: VMBinding> CopyContext for GenCopyCopyContext<VM> {
         _bytes: usize,
         _semantics: crate::AllocationSemantics,
     ) {
-        forwarding_word::clear_forwarding_bits::<VM>(obj);
+        object_forwarding::clear_forwarding_bits::<VM>(obj);
         if !super::NO_SLOW && super::ACTIVE_BARRIER == BarrierSelector::ObjectBarrier {
-            store_atomic(super::LOGGING_META, obj.to_address(), 0b1);
+            store_metadata::<VM>(
+                VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC,
+                obj,
+                0b1,
+                None,
+                Some(Ordering::SeqCst),
+            );
         }
     }
 }

src/plan/gencopy/global.rs

@@ -1,8 +1,5 @@
+use super::gc_work::{GenCopyCopyContext, GenCopyMatureProcessEdges, GenCopyNurseryProcessEdges};
 use super::mutator::ALLOCATOR_MAPPING;
-use super::{
-    gc_work::{GenCopyCopyContext, GenCopyMatureProcessEdges, GenCopyNurseryProcessEdges},
-    LOGGING_META,
-};
 use crate::plan::global::BasePlan;
 use crate::plan::global::CommonPlan;
 use crate::plan::global::GcStatus;
@@ -14,17 +11,17 @@ use crate::policy::space::Space;
 use crate::scheduler::gc_work::*;
 use crate::scheduler::*;
 use crate::util::alloc::allocators::AllocatorSelector;
-use crate::util::conversions;
 use crate::util::heap::layout::heap_layout::Mmapper;
 use crate::util::heap::layout::heap_layout::VMMap;
 use crate::util::heap::layout::vm_layout_constants::{HEAP_END, HEAP_START};
 use crate::util::heap::HeapMeta;
 use crate::util::heap::VMRequest;
+use crate::util::metadata::side_metadata::{SideMetadataContext, SideMetadataSanity};
 use crate::util::options::UnsafeOptionsWrapper;
 #[cfg(feature = "sanity")]
 use crate::util::sanity::sanity_checker::*;
-use crate::util::side_metadata::{SideMetadataContext, SideMetadataSanity};
 use crate::util::VMWorkerThread;
+use crate::util::{conversions, metadata};
 use crate::vm::*;
 use crate::{mmtk::MMTK, plan::barriers::BarrierSelector};
 use enum_map::EnumMap;
@@ -203,7 +200,7 @@ impl<VM: VMBinding> GenCopy<VM> {
     ) -> Self {
         let mut heap = HeapMeta::new(HEAP_START, HEAP_END);
         let gencopy_specs = if super::ACTIVE_BARRIER == BarrierSelector::ObjectBarrier {
-            vec![LOGGING_META]
+            metadata::extract_side_metadata(&[VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC])
         } else {
             vec![]
         };

src/plan/gencopy/mod.rs

@@ -23,18 +23,3 @@ pub const FULL_NURSERY_GC: bool = false;
 pub const NO_SLOW: bool = false;
 
 pub use self::global::GENCOPY_CONSTRAINTS;
-
-#[cfg(feature = "side_gc_header")]
-use crate::util::gc_byte;
-use crate::util::side_metadata::*;
-
-const LOGGING_META: SideMetadataSpec = SideMetadataSpec {
-    scope: SideMetadataScope::Global,
-    #[cfg(not(feature = "side_gc_header"))]
-    offset: GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize(),
-    #[cfg(feature = "side_gc_header")]
-    offset: GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize()
-        + metadata_address_range_size(gc_byte::SIDE_GC_BYTE_SPEC),
-    log_num_of_bits: 0,
-    log_min_obj_size: 3,
-};

src/plan/gencopy/mutator.rs

@@ -7,7 +7,7 @@ use crate::plan::AllocationSemantics as AllocationType;
 use crate::util::alloc::allocators::{AllocatorSelector, Allocators};
 use crate::util::alloc::BumpAllocator;
 use crate::util::{VMMutatorThread, VMWorkerThread};
-use crate::vm::VMBinding;
+use crate::vm::{ObjectModel, VMBinding};
 use crate::MMTK;
 use enum_map::enum_map;
 use enum_map::EnumMap;
@@ -59,7 +59,7 @@ pub fn create_gencopy_mutator<VM: VMBinding>(
         allocators: Allocators::<VM>::new(mutator_tls, &*mmtk.plan, &config.space_mapping),
         barrier: box ObjectRememberingBarrier::<GenCopyNurseryProcessEdges<VM>>::new(
             mmtk,
-            super::LOGGING_META,
+            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC,
         ),
         mutator_tls,
         config,

src/plan/global.rs

@@ -19,10 +19,10 @@ use crate::util::heap::layout::heap_layout::VMMap;
 use crate::util::heap::layout::map::Map;
 use crate::util::heap::HeapMeta;
 use crate::util::heap::VMRequest;
+use crate::util::metadata::side_metadata::SideMetadataSanity;
+use crate::util::metadata::side_metadata::SideMetadataSpec;
 use crate::util::options::PlanSelector;
 use crate::util::options::{Options, UnsafeOptionsWrapper};
-use crate::util::side_metadata::SideMetadataSanity;
-use crate::util::side_metadata::SideMetadataSpec;
 use crate::util::statistics::stats::Stats;
 use crate::util::{Address, ObjectReference};
 use crate::util::{VMMutatorThread, VMWorkerThread};

src/plan/marksweep/global.rs

@@ -17,10 +17,10 @@ use crate::util::heap::layout::heap_layout::Mmapper;
 use crate::util::heap::layout::heap_layout::VMMap;
 use crate::util::heap::layout::vm_layout_constants::{HEAP_END, HEAP_START};
 use crate::util::heap::HeapMeta;
+use crate::util::metadata::side_metadata::{SideMetadataContext, SideMetadataSanity};
 use crate::util::options::UnsafeOptionsWrapper;
 #[cfg(feature = "sanity")]
 use crate::util::sanity::sanity_checker::*;
-use crate::util::side_metadata::{SideMetadataContext, SideMetadataSanity};
 use crate::util::VMWorkerThread;
 use crate::vm::VMBinding;
 use std::sync::Arc;

src/plan/nogc/global.rs

@@ -15,9 +15,9 @@ use crate::util::heap::layout::vm_layout_constants::{HEAP_END, HEAP_START};
 use crate::util::heap::HeapMeta;
 #[allow(unused_imports)]
 use crate::util::heap::VMRequest;
+use crate::util::metadata::side_metadata::{SideMetadataContext, SideMetadataSanity};
 use crate::util::opaque_pointer::*;
 use crate::util::options::UnsafeOptionsWrapper;
-use crate::util::side_metadata::{SideMetadataContext, SideMetadataSanity};
 use crate::vm::VMBinding;
 use enum_map::EnumMap;
 use std::sync::Arc;

src/plan/plan_constraints.rs

@@ -13,8 +13,6 @@ pub struct PlanConstraints {
     pub gc_header_words: usize,
     pub num_specialized_scans: usize,
     pub max_non_los_copy_bytes: usize,
-    pub needs_log_bit_in_header: bool,
-    pub needs_log_bit_in_header_num: usize,
     pub barrier: BarrierSelector,
     // the following seems unused for now
     pub needs_linear_scan: bool,
@@ -31,8 +29,6 @@ impl PlanConstraints {
             gc_header_words: 0,
             num_specialized_scans: 0,
             max_non_los_copy_bytes: MAX_INT,
-            needs_log_bit_in_header: false,
-            needs_log_bit_in_header_num: 0,
             needs_linear_scan: SUPPORT_CARD_SCANNING || LAZY_SWEEP,
             needs_concurrent_workers: false,
             generate_gc_trace: false,

src/plan/semispace/gc_work.rs

@@ -5,7 +5,7 @@ use crate::policy::space::Space;
 use crate::scheduler::gc_work::*;
 use crate::scheduler::WorkerLocal;
 use crate::util::alloc::{Allocator, BumpAllocator};
-use crate::util::forwarding_word;
+use crate::util::object_forwarding;
 use crate::util::opaque_pointer::*;
 use crate::util::{Address, ObjectReference};
 use crate::vm::VMBinding;
@@ -51,7 +51,7 @@ impl<VM: VMBinding> CopyContext for SSCopyContext<VM> {
         _bytes: usize,
         _semantics: crate::AllocationSemantics,
     ) {
-        forwarding_word::clear_forwarding_bits::<VM>(obj);
+        object_forwarding::clear_forwarding_bits::<VM>(obj);
     }
 }
 

src/plan/semispace/global.rs

@@ -18,11 +18,11 @@ use crate::util::heap::layout::heap_layout::VMMap;
 use crate::util::heap::layout::vm_layout_constants::{HEAP_END, HEAP_START};
 use crate::util::heap::HeapMeta;
 use crate::util::heap::VMRequest;
+use crate::util::metadata::side_metadata::{SideMetadataContext, SideMetadataSanity};
 use crate::util::opaque_pointer::VMWorkerThread;
 use crate::util::options::UnsafeOptionsWrapper;
 #[cfg(feature = "sanity")]
 use crate::util::sanity::sanity_checker::*;
-use crate::util::side_metadata::{SideMetadataContext, SideMetadataSanity};
 use crate::{plan::global::BasePlan, vm::VMBinding};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;

src/policy/copyspace.rs

@@ -3,12 +3,13 @@ use crate::plan::{AllocationSemantics, CopyContext};
 use crate::policy::space::SpaceOptions;
 use crate::policy::space::{CommonSpace, Space, SFT};
 use crate::util::constants::CARD_META_PAGES_PER_REGION;
-use crate::util::forwarding_word as ForwardingWord;
 use crate::util::heap::layout::heap_layout::{Mmapper, VMMap};
 use crate::util::heap::HeapMeta;
 use crate::util::heap::VMRequest;
 use crate::util::heap::{MonotonePageResource, PageResource};
-use crate::util::side_metadata::{SideMetadataContext, SideMetadataSpec};
+use crate::util::metadata::extract_side_metadata;
+use crate::util::metadata::side_metadata::{SideMetadataContext, SideMetadataSpec};
+use crate::util::object_forwarding;
 use crate::util::{Address, ObjectReference};
 use crate::vm::*;
 use libc::{mprotect, PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE};
@@ -28,7 +29,7 @@ impl<VM: VMBinding> SFT for CopySpace<VM> {
         self.get_name()
     }
     fn is_live(&self, object: ObjectReference) -> bool {
-        !self.from_space() || ForwardingWord::is_forwarded::<VM>(object)
+        !self.from_space() || object_forwarding::is_forwarded::<VM>(object)
     }
     fn is_movable(&self) -> bool {
         true
@@ -75,6 +76,10 @@ impl<VM: VMBinding> CopySpace<VM> {
         mmapper: &'static Mmapper,
         heap: &mut HeapMeta,
     ) -> Self {
+        let local_specs = extract_side_metadata(&[
+            VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC,
+            VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC,
+        ]);
         let common = CommonSpace::new(
             SpaceOptions {
                 name,
@@ -84,7 +89,7 @@ impl<VM: VMBinding> CopySpace<VM> {
                 vmrequest,
                 side_metadata_specs: SideMetadataContext {
                     global: global_side_metadata_specs,
-                    local: vec![],
+                    local: local_specs,
                 },
             },
             vm_map,
@@ -136,21 +141,21 @@ impl<VM: VMBinding> CopySpace<VM> {
             return object;
         }
         trace!("attempting to forward");
-        let forwarding_status = ForwardingWord::attempt_to_forward::<VM>(object);
+        let forwarding_status = object_forwarding::attempt_to_forward::<VM>(object);
         trace!("checking if object is being forwarded");
-        if ForwardingWord::state_is_forwarded_or_being_forwarded(forwarding_status) {
+        if object_forwarding::state_is_forwarded_or_being_forwarded(forwarding_status) {
             trace!("... yes it is");
             let new_object =
-                ForwardingWord::spin_and_get_forwarded_object::<VM>(object, forwarding_status);
+                object_forwarding::spin_and_get_forwarded_object::<VM>(object, forwarding_status);
             trace!("Returning");
             new_object
         } else {
             trace!("... no it isn't. Copying");
             let new_object =
-                ForwardingWord::forward_object::<VM, _>(object, semantics, copy_context);
+                object_forwarding::forward_object::<VM, _>(object, semantics, copy_context);
             trace!("Forwarding pointer");
             trace.process_node(new_object);
-            trace!("Copying [{:?} -> {:?}]", object, new_object);
+            trace!("Copied [{:?} -> {:?}]", object, new_object);
             new_object
         }
     }