8350973: Add memory overhead counter to NMT

src/hotspot/share/memory/arena.cpp

@@ -282,7 +282,7 @@ void Arena::set_size_in_bytes(size_t size) {
   if (_size_in_bytes != size) {
     ssize_t delta = size - size_in_bytes();
     _size_in_bytes = size;
-    MemTracker::record_arena_size_change(delta, _mem_tag);
+    MemTracker::record_arena_size_change(delta, 0, _mem_tag);
   }
 }
 

src/hotspot/share/nmt/mallocHeader.hpp

@@ -127,7 +127,7 @@ class MallocHeader {
 
   inline MallocHeader(size_t size, MemTag mem_tag, uint32_t mst_marker);
 
-  inline static size_t malloc_overhead() { return sizeof(MallocHeader) + sizeof(uint16_t); }
+  inline static size_t nmt_overhead() { return sizeof(MallocHeader) + sizeof(uint16_t); }
   inline size_t size()  const { return _size; }
   inline MemTag mem_tag() const { return _mem_tag; }
   inline uint32_t mst_marker() const { return _mst_marker; }

src/hotspot/share/nmt/mallocSiteTable.hpp

@@ -41,8 +41,8 @@ class MallocSite : public AllocationSite {
   MallocSite(const NativeCallStack& stack, MemTag mem_tag) :
     AllocationSite(stack, mem_tag) {}
 
-  void allocate(size_t size)      { _c.allocate(size);   }
-  void deallocate(size_t size)    { _c.deallocate(size); }
+  void request(size_t requested, size_t allocated)      { _c.request(requested, allocated);   }
+  void deallocate(size_t requested, size_t allocated)    { _c.deallocate(requested, allocated); }
 
   // Memory allocated from this code path
   size_t size()  const { return _c.size(); }
@@ -83,8 +83,8 @@ class MallocSiteHashtableEntry : public CHeapObj<mtNMT> {
   inline MallocSite* data()             { return &_malloc_site; }
 
   // Allocation/deallocation on this allocation site
-  inline void allocate(size_t size)   { _malloc_site.allocate(size);   }
-  inline void deallocate(size_t size) { _malloc_site.deallocate(size); }
+  inline void request(size_t requested, size_t allocated)   { _malloc_site.request(requested, allocated);   }
+  inline void deallocate(size_t requested, size_t allocated) { _malloc_site.deallocate(requested, allocated); }
   // Memory counters
   inline size_t size() const  { return _malloc_site.size();  }
   inline size_t count() const { return _malloc_site.count(); }
@@ -146,19 +146,19 @@ class MallocSiteTable : AllStatic {
   // Return false only occurs under rare scenarios:
   //  1. out of memory
   //  2. overflow hash bucket
-  static inline bool allocation_at(const NativeCallStack& stack, size_t size,
+  static inline bool allocation_at(const NativeCallStack& stack, size_t requested, size_t allocated,
       uint32_t* marker, MemTag mem_tag) {
     MallocSite* site = lookup_or_add(stack, marker, mem_tag);
-    if (site != nullptr) site->allocate(size);
+    if (site != nullptr) site->request(requested, allocated);
     return site != nullptr;
   }
 
   // Record memory deallocation. marker indicates where the allocation
   // information was recorded.
-  static inline bool deallocation_at(size_t size, uint32_t marker) {
+  static inline bool deallocation_at(size_t requested, size_t allocated, uint32_t marker) {
     MallocSite* site = malloc_site(marker);
     if (site != nullptr) {
-      site->deallocate(size);
+      site->deallocate(requested, allocated);
       return true;
     }
     return false;

src/hotspot/share/nmt/mallocTracker.cpp

@@ -66,15 +66,17 @@ void MallocMemorySnapshot::copy_to(MallocMemorySnapshot* s) {
   // buffer in make_adjustment().
   ThreadCritical tc;
   s->_all_mallocs = _all_mallocs;
-  size_t total_size = 0;
+  size_t total_requested = 0;
+  size_t total_allocated = 0;
   size_t total_count = 0;
   for (int index = 0; index < mt_number_of_tags; index ++) {
     s->_malloc[index] = _malloc[index];
-    total_size += s->_malloc[index].malloc_size();
+    total_requested += s->_malloc[index].malloc_requested();
+    total_allocated += s->_malloc[index].malloc_allocated();
     total_count += s->_malloc[index].malloc_count();
   }
   // malloc counters may be updated concurrently
-  s->_all_mallocs.set_size_and_count(total_size, total_count);
+  s->_all_mallocs.set_size_and_count(total_requested, total_allocated, total_count);
 }
 
 // Total malloc'd memory used by arenas
@@ -91,8 +93,8 @@ size_t MallocMemorySnapshot::total_arena() const {
 void MallocMemorySnapshot::make_adjustment() {
   size_t arena_size = total_arena();
   int chunk_idx = NMTUtil::tag_to_index(mtChunk);
-  _malloc[chunk_idx].record_free(arena_size);
-  _all_mallocs.deallocate(arena_size);
+  _malloc[chunk_idx].record_free(arena_size, 0);
+  _all_mallocs.deallocate(arena_size, 0);
 }
 
 void MallocMemorySummary::initialize() {
@@ -166,21 +168,21 @@ bool MallocTracker::initialize(NMT_TrackingLevel level) {
 }
 
 // Record a malloc memory allocation
-void* MallocTracker::record_malloc(void* malloc_base, size_t size, MemTag mem_tag,
+void* MallocTracker::record_malloc(void* ptr, size_t requested, size_t allocated, MemTag mem_tag,
   const NativeCallStack& stack)
 {
   assert(MemTracker::enabled(), "precondition");
-  assert(malloc_base != nullptr, "precondition");
+  assert(ptr != nullptr, "precondition");
 
-  MallocMemorySummary::record_malloc(size, mem_tag);
+  MallocMemorySummary::record_malloc(requested, allocated, mem_tag);
   uint32_t mst_marker = 0;
   if (MemTracker::tracking_level() == NMT_detail) {
-    MallocSiteTable::allocation_at(stack, size, &mst_marker, mem_tag);
+    MallocSiteTable::allocation_at(stack, requested, allocated, &mst_marker, mem_tag);
   }
 
   // Uses placement global new operator to initialize malloc header
-  MallocHeader* const header = ::new (malloc_base)MallocHeader(size, mem_tag, mst_marker);
-  void* const memblock = (void*)((char*)malloc_base + sizeof(MallocHeader));
+  MallocHeader* const header = ::new (ptr)MallocHeader(requested, mem_tag, mst_marker);
+  void* const memblock = (void*)((char*)ptr + sizeof(MallocHeader));
 
   // The alignment check: 8 bytes alignment for 32 bit systems.
   //                      16 bytes alignment for 64-bit systems.
@@ -198,23 +200,23 @@ void* MallocTracker::record_malloc(void* malloc_base, size_t size, MemTag mem_ta
   return memblock;
 }
 
-void* MallocTracker::record_free_block(void* memblock) {
+void* MallocTracker::record_free_block(void* ptr, size_t allocated) {
   assert(MemTracker::enabled(), "Sanity");
-  assert(memblock != nullptr, "precondition");
+  assert(ptr != nullptr, "precondition");
 
-  MallocHeader* header = MallocHeader::resolve_checked(memblock);
+  MallocHeader* header = MallocHeader::resolve_checked(ptr);
 
-  deaccount(header->free_info());
+  deaccount(header->free_info(), allocated);
 
   header->mark_block_as_dead();
 
   return (void*)header;
 }
 
-void MallocTracker::deaccount(MallocHeader::FreeInfo free_info) {
-  MallocMemorySummary::record_free(free_info.size, free_info.mem_tag);
+void MallocTracker::deaccount(MallocHeader::FreeInfo free_info, size_t allocated) {
+  MallocMemorySummary::record_free(free_info.size, allocated, free_info.mem_tag);
   if (MemTracker::tracking_level() == NMT_detail) {
-    MallocSiteTable::deallocation_at(free_info.size, free_info.mst_marker);
+    MallocSiteTable::deallocation_at(free_info.size, allocated, free_info.mst_marker);
   }
 }
 

src/hotspot/share/nmt/mallocTracker.hpp

@@ -38,13 +38,14 @@ struct malloclimit;
 
 /*
  * This counter class counts memory allocation and deallocation,
- * records total memory allocation size and number of allocations.
+ * records total memory bytes requested and allocated and number of allocations.
  * The counters are updated atomically.
  */
 class MemoryCounter {
  private:
   volatile size_t   _count;
-  volatile size_t   _size;
+  volatile size_t   _requested; // how much bytes we asked for
+  volatile size_t   _allocated; // how much bytes the os malloc actually allocated (usually it's more)
 
   // Peak size and count. Note: Peak count is the count at the point
   // peak size was reached, not the absolute highest peak count.
@@ -53,41 +54,51 @@ class MemoryCounter {
   void update_peak(size_t size, size_t cnt);
 
  public:
-  MemoryCounter() : _count(0), _size(0), _peak_count(0), _peak_size(0) {}
+  MemoryCounter() : _count(0), _requested(0), _allocated(0), _peak_count(0), _peak_size(0) {}
 
-  inline void set_size_and_count(size_t size, size_t count) {
-    _size = size;
+  inline void set_size_and_count(size_t requested, size_t allocated, size_t count) {
+    _requested = requested;
+    _allocated = allocated;
     _count = count;
-    update_peak(size, count);
+    update_peak(requested, count);
   }
 
-  inline void allocate(size_t sz) {
+  inline void request(size_t requested, size_t allocated) {
     size_t cnt = Atomic::add(&_count, size_t(1), memory_order_relaxed);
-    if (sz > 0) {
-      size_t sum = Atomic::add(&_size, sz, memory_order_relaxed);
+    if (requested > 0) {
+      size_t sum = Atomic::add(&_requested, requested, memory_order_relaxed);
       update_peak(sum, cnt);
     }
+    if (allocated > 0) {
+      Atomic::add(&_allocated, allocated, memory_order_relaxed);
+    }
   }
 
-  inline void deallocate(size_t sz) {
+  inline void deallocate(size_t requested, size_t allocated) {
     assert(count() > 0, "Nothing allocated yet");
-    assert(size() >= sz, "deallocation > allocated");
+    assert(size() >= requested, "deallocation > allocated");
     Atomic::dec(&_count, memory_order_relaxed);
-    if (sz > 0) {
-      Atomic::sub(&_size, sz, memory_order_relaxed);
+    if (requested > 0) {
+      Atomic::sub(&_requested, requested, memory_order_relaxed);
+    }
+    if (allocated > 0) {
+      Atomic::sub(&_allocated, allocated, memory_order_relaxed);
     }
   }
 
-  inline void resize(ssize_t sz) {
-    if (sz != 0) {
-      assert(sz >= 0 || size() >= size_t(-sz), "Must be");
-      size_t sum = Atomic::add(&_size, size_t(sz), memory_order_relaxed);
+  inline void resize(ssize_t requested, ssize_t allocated) {
+    if (requested != 0) {
+      assert(requested >= 0 || size() >= size_t(-requested), "Must be");
+      size_t sum = Atomic::add(&_requested, size_t(requested), memory_order_relaxed);
       update_peak(sum, _count);
+      Atomic::add(&_allocated, size_t(allocated), memory_order_relaxed);
     }
   }
 
   inline size_t count() const { return Atomic::load(&_count); }
-  inline size_t size()  const { return Atomic::load(&_size);  }
+  inline size_t size()  const { return Atomic::load(&_requested);  }
+  inline size_t requested()  const { return Atomic::load(&_requested);  }
+  inline size_t allocated()  const { return Atomic::load(&_allocated);  }
 
   inline size_t peak_count() const {
     return Atomic::load(&_peak_count);
@@ -111,27 +122,28 @@ class MallocMemory {
  public:
   MallocMemory() { }
 
-  inline void record_malloc(size_t sz) {
-    _malloc.allocate(sz);
+  inline void record_malloc(size_t requested, size_t allocated) {
+    _malloc.request(requested, allocated);
   }
 
-  inline void record_free(size_t sz) {
-    _malloc.deallocate(sz);
+  inline void record_free(size_t requested, size_t allocated) {
+    _malloc.deallocate(requested, allocated);
   }
 
   inline void record_new_arena() {
-    _arena.allocate(0);
+    _arena.request(0, 0);
   }
 
   inline void record_arena_free() {
-    _arena.deallocate(0);
+    _arena.deallocate(0, 0);
   }
 
-  inline void record_arena_size_change(ssize_t sz) {
-    _arena.resize(sz);
+  inline void record_arena_size_change(ssize_t requested, size_t allocated) {
+    _arena.resize(requested, allocated);
   }
 
-  inline size_t malloc_size()  const { return _malloc.size(); }
+  inline size_t malloc_requested()  const { return _malloc.requested(); }
+  inline size_t malloc_allocated()  const { return _malloc.allocated(); }
   inline size_t malloc_peak_size()  const { return _malloc.peak_size(); }
   inline size_t malloc_count() const { return _malloc.count();}
   inline size_t arena_size()   const { return _arena.size();  }
@@ -165,8 +177,20 @@ class MallocMemorySnapshot {
     return &_malloc[index];
   }
 
+  inline size_t requested() const {
+    return _all_mallocs.requested();
+  }
+
+  inline size_t allocated() const {
+    return _all_mallocs.allocated();
+  }
+
+  inline size_t nmt_overhead() const {
+    return _all_mallocs.count() * MallocHeader::nmt_overhead();
+  }
+
   inline size_t malloc_overhead() const {
-    return _all_mallocs.count() * MallocHeader::malloc_overhead();
+    return _all_mallocs.allocated() - _all_mallocs.requested();
   }
 
   // Total malloc invocation count
@@ -176,7 +200,7 @@ class MallocMemorySnapshot {
 
   // Total malloc'd memory amount
   size_t total() const {
-    return _all_mallocs.size() + malloc_overhead() + total_arena();
+    return _all_mallocs.size() + nmt_overhead() + total_arena();
   }
 
   // Total peak malloc
@@ -219,14 +243,14 @@ class MallocMemorySummary : AllStatic {
  public:
    static void initialize();
 
-   static inline void record_malloc(size_t size, MemTag mem_tag) {
-     as_snapshot()->by_type(mem_tag)->record_malloc(size);
-     as_snapshot()->_all_mallocs.allocate(size);
+   static inline void record_malloc(size_t requested, size_t allocated, MemTag mem_tag) {
+     as_snapshot()->by_type(mem_tag)->record_malloc(requested, allocated);
+     as_snapshot()->_all_mallocs.request(requested, allocated);
    }
 
-   static inline void record_free(size_t size, MemTag mem_tag) {
-     as_snapshot()->by_type(mem_tag)->record_free(size);
-     as_snapshot()->_all_mallocs.deallocate(size);
+   static inline void record_free(size_t requested, size_t allocated, MemTag mem_tag) {
+     as_snapshot()->by_type(mem_tag)->record_free(requested, allocated);
+     as_snapshot()->_all_mallocs.deallocate(requested, allocated);
    }
 
    static inline void record_new_arena(MemTag mem_tag) {
@@ -237,8 +261,8 @@ class MallocMemorySummary : AllStatic {
      as_snapshot()->by_type(mem_tag)->record_arena_free();
    }
 
-   static inline void record_arena_size_change(ssize_t size, MemTag mem_tag) {
-     as_snapshot()->by_type(mem_tag)->record_arena_size_change(size);
+   static inline void record_arena_size_change(ssize_t requested, size_t allocated, MemTag mem_tag) {
+     as_snapshot()->by_type(mem_tag)->record_arena_size_change(requested, allocated);
    }
 
    static void snapshot(MallocMemorySnapshot* s) {
@@ -248,7 +272,7 @@ class MallocMemorySummary : AllStatic {
 
    // The memory used by malloc tracking headers
    static inline size_t tracking_overhead() {
-     return as_snapshot()->malloc_overhead();
+     return as_snapshot()->nmt_overhead();
    }
 
   static MallocMemorySnapshot* as_snapshot() {
@@ -269,7 +293,7 @@ class MallocTracker : AllStatic {
 
   // The overhead that is incurred by switching on NMT (we need, per malloc allocation,
   // space for header and 16-bit footer)
-  static inline size_t overhead_per_malloc() { return MallocHeader::malloc_overhead(); }
+  static inline size_t overhead_per_malloc() { return MallocHeader::nmt_overhead(); }
 
   // Parameter name convention:
   // memblock :   the beginning address for user data
@@ -280,14 +304,14 @@ class MallocTracker : AllStatic {
   //
 
   // Record  malloc on specified memory block
-  static void* record_malloc(void* malloc_base, size_t size, MemTag mem_tag,
+  static void* record_malloc(void* ptr, size_t requested, size_t allocated, MemTag mem_tag,
     const NativeCallStack& stack);
 
   // Given a block returned by os::malloc() or os::realloc():
   // deaccount block from NMT, mark its header as dead and return pointer to header.
-  static void* record_free_block(void* memblock);
+  static void* record_free_block(void* ptr, size_t allocated);
   // Given the free info from a block, de-account block from NMT.
-  static void deaccount(MallocHeader::FreeInfo free_info);
+  static void deaccount(MallocHeader::FreeInfo free_info, size_t allocated);
 
   static inline void record_new_arena(MemTag mem_tag) {
     MallocMemorySummary::record_new_arena(mem_tag);
@@ -297,8 +321,8 @@ class MallocTracker : AllStatic {
     MallocMemorySummary::record_arena_free(mem_tag);
   }
 
-  static inline void record_arena_size_change(ssize_t size, MemTag mem_tag) {
-    MallocMemorySummary::record_arena_size_change(size, mem_tag);
+  static inline void record_arena_size_change(ssize_t requested, size_t allocated, MemTag mem_tag) {
+    MallocMemorySummary::record_arena_size_change(requested, allocated, mem_tag);
   }
 
   // MallocLimt: Given an allocation size s, check if mallocing this much

src/hotspot/share/nmt/mallocTracker.inline.hpp

@@ -53,7 +53,7 @@ inline bool MallocMemorySummary::check_exceeds_limit(size_t s, MemTag mem_tag) {
       l = MallocLimitHandler::category_limit(mem_tag);
       if (l->sz > 0) {
         const MallocMemory* mm = as_snapshot()->by_type(mem_tag);
-        size_t so_far = mm->malloc_size() + mm->arena_size();
+        size_t so_far = mm->malloc_requested() + mm->arena_size();
         if ((so_far + s) > l->sz) {
           return category_limit_reached(mem_tag, s, so_far, l);
         }