Bug 1934856 - Part 2: Use buffer allocator for JSObject elements r=sfink,jandem

Differential Revision: https://phabricator.services.mozilla.com/D230956

Author: jonco3

js/src/gc/GCEnum.h

@@ -108,7 +108,6 @@ enum class GCAbortReason {
 #define JS_FOR_EACH_INTERNAL_MEMORY_USE(_) \
   _(ArrayBufferContents)                   \
   _(StringContents)                        \
-  _(ObjectElements)                        \
   _(ScriptPrivateData)                     \
   _(MapObjectData)                         \
   _(BigIntDigits)                          \

js/src/gc/Marking.cpp

@@ -1671,6 +1671,11 @@ scan_obj: {
     BufferAllocator::MarkTenuredAlloc(slots);
   }
 
+  if (nobj->hasDynamicElements()) {
+    void* elements = nobj->getUnshiftedElementsHeader();
+    BufferAllocator::MarkTenuredAlloc(elements);
+  }
+
   if (!nobj->hasEmptyElements()) {
     base = nobj->getDenseElements();
     kind = SlotsOrElementsKind::Elements;

js/src/gc/Tenuring.cpp

@@ -930,8 +930,7 @@ size_t js::gc::TenuringTracer::moveElements(NativeObject* dst,
   /* TODO Bug 874151: Prefer to put element data inline if we have space. */
 
   Nursery::WasBufferMoved result =
-      nursery().maybeMoveNurseryOrMallocBufferOnPromotion(
-          &unshiftedHeader, dst, allocSize, MemoryUse::ObjectElements);
+      nursery().maybeMoveBufferOnPromotion(&unshiftedHeader, dst, allocSize);
   if (result == Nursery::BufferNotMoved) {
     return 0;
   }

js/src/jit-test/lib/pretenure.js

@@ -6,7 +6,7 @@ const is64bit = getBuildConfiguration("pointer-byte-size") === 8;
 const nurseryCount = is64bit ? 25000 : 50000;
 
 // Count of objects that will exceed the tenured heap collection threshold.
-const tenuredCount = is64bit ? 300000 : 600000;
+const tenuredCount = is64bit ? 400000 : 800000;
 
 function setupPretenureTest() {
   // The test requires that baseline is enabled and is not bypassed with

js/src/jit-test/tests/arrays/shrink-large-array.js

@@ -0,0 +1,8 @@
+// Shrink a large dense elements allocation of ~2MB to half its size.
+const sizeMB = 2;
+const elementCount = (sizeMB * 1024 * 1024) / 8;
+const array = new Array(elementCount);
+for (let i = 0; i < elementCount; i++) {
+  array[i] = i;
+}
+array.length = elementCount / 2;

js/src/jit-test/tests/gc/pretenured-operations.js

@@ -35,9 +35,9 @@ check(() => { return Object(); });
 
 // Array construction.
 check(() => { return Array(); });
-check(() => { return Array(100); });
+check(() => { return Array(150); });
 check(() => { return new Array(); });
-check(() => { return new Array(100); });
+check(() => { return new Array(150); });
 
 // DOM Allocations
 let fdo = new FakeDOMObject();

js/src/jit-test/tests/heap-analysis/byteSize-of-object.js

@@ -36,11 +36,11 @@ assertEq(tByteSize({ w: 1, x: 2, y: 3, z:4 }),          s(48,  56));
 assertEq(tByteSize({ w: 1, x: 2, y: 3, z:4, a: 5 }),    s(80,  88));
 
 // Try objects with only indexed properties.
-assertEq(tByteSize({ 0:0 }),                            s(80,  88));
-assertEq(tByteSize({ 0:0, 1:1 }),                       s(80,  88));
-assertEq(tByteSize({ 0:0, 1:1, 2:2 }),                  s(96,  104));
-assertEq(tByteSize({ 0:0, 1:1, 2:2, 3:3 }),             s(96,  104));
-assertEq(tByteSize({ 0:0, 1:1, 2:2, 3:3, 4:4 }),        s(144, 152));
+assertEq(tByteSize({ 0:0 }),                            s(88,  96));
+assertEq(tByteSize({ 0:0, 1:1 }),                       s(88,  96));
+assertEq(tByteSize({ 0:0, 1:1, 2:2 }),                  s(104, 112));
+assertEq(tByteSize({ 0:0, 1:1, 2:2, 3:3 }),             s(104, 112));
+assertEq(tByteSize({ 0:0, 1:1, 2:2, 3:3, 4:4 }),        s(136, 144));
 
 // Mix indexed and named properties, exploring each combination of the size
 // classes above.
@@ -49,15 +49,15 @@ assertEq(tByteSize({ 0:0, 1:1, 2:2, 3:3, 4:4 }),        s(144, 152));
 // changes above: for example, with one named property, the objects with three
 // and five indexed properties are in different size classes; but with three
 // named properties, there's no break there.
-assertEq(tByteSize({ w:1,                     0:0                     }),  s(80,  88));
-assertEq(tByteSize({ w:1,                     0:0, 1:1, 2:2           }),  s(96,  104));
-assertEq(tByteSize({ w:1,                     0:0, 1:1, 2:2, 3:3, 4:4 }),  s(144, 152));
-assertEq(tByteSize({ w:1, x:2, y:3,           0:0                     }),  s(96,  104));
-assertEq(tByteSize({ w:1, x:2, y:3,           0:0, 1:1, 2:2           }),  s(128, 136));
-assertEq(tByteSize({ w:1, x:2, y:3,           0:0, 1:1, 2:2, 3:3, 4:4 }),  s(144, 152));
-assertEq(tByteSize({ w:1, x:2, y:3, z:4, a:6, 0:0                     }),  s(128, 136));
-assertEq(tByteSize({ w:1, x:2, y:3, z:4, a:6, 0:0, 1:1, 2:2           }),  s(128, 136));
-assertEq(tByteSize({ w:1, x:2, y:3, z:4, a:6, 0:0, 1:1, 2:2, 3:3, 4:4 }),  s(176, 184));
+assertEq(tByteSize({ w:1,                     0:0                     }),  s(88,  96));
+assertEq(tByteSize({ w:1,                     0:0, 1:1, 2:2           }),  s(104, 112));
+assertEq(tByteSize({ w:1,                     0:0, 1:1, 2:2, 3:3, 4:4 }),  s(136, 144));
+assertEq(tByteSize({ w:1, x:2, y:3,           0:0                     }),  s(104, 112));
+assertEq(tByteSize({ w:1, x:2, y:3,           0:0, 1:1, 2:2           }),  s(136, 144));
+assertEq(tByteSize({ w:1, x:2, y:3,           0:0, 1:1, 2:2, 3:3, 4:4 }),  s(136, 144));
+assertEq(tByteSize({ w:1, x:2, y:3, z:4, a:6, 0:0                     }),  s(136, 144));
+assertEq(tByteSize({ w:1, x:2, y:3, z:4, a:6, 0:0, 1:1, 2:2           }),  s(136, 144));
+assertEq(tByteSize({ w:1, x:2, y:3, z:4, a:6, 0:0, 1:1, 2:2, 3:3, 4:4 }),  s(168, 176));
 
 // Check various lengths of array.
 assertEq(tByteSize([]),                                 s(80,  88));

js/src/vm/JSObject-inl.h

@@ -105,18 +105,6 @@ inline void JSObject::finalize(JS::GCContext* gcx) {
   if (clasp->hasFinalize()) {
     clasp->doFinalize(gcx, this);
   }
-
-  if (!objShape->isNative()) {
-    return;
-  }
-
-  js::NativeObject* nobj = &as<js::NativeObject>();
-  if (nobj->hasDynamicElements()) {
-    js::ObjectElements* elements = nobj->getElementsHeader();
-    size_t size = elements->numAllocatedElements() * sizeof(js::HeapSlot);
-    gcx->free_(this, nobj->getUnshiftedElementsHeader(), size,
-               js::MemoryUse::ObjectElements);
-  }
 }
 
 inline bool JSObject::isQualifiedVarObj() const {

js/src/vm/JSObject.cpp

@@ -1035,21 +1035,17 @@ bool NativeObject::prepareForSwap(JSContext* cx,
     size_t count = elements->numAllocatedElements();
     size_t size = count * sizeof(HeapSlot);
 
-    if (isTenured()) {
-      RemoveCellMemory(this, size, MemoryUse::ObjectElements);
-    } else if (cx->nursery().isInside(allocatedElements)) {
+    if (ChunkPtrIsInsideNursery(allocatedElements)) {
       // Move nursery allocated elements in case they end up in a tenured
       // object.
-      ObjectElements* newElements =
-          reinterpret_cast<ObjectElements*>(js_pod_malloc<HeapSlot>(count));
+      ObjectElements* newElements = static_cast<ObjectElements*>(
+          AllocBuffer(cx->zone(), size, !isTenured()));
       if (!newElements) {
         return false;
       }
 
       memmove(newElements, elements, size);
       elements_ = newElements->elements();
-    } else {
-      cx->nursery().removeMallocedBuffer(allocatedElements, size);
     }
     MOZ_ASSERT(hasDynamicElements());
   }
@@ -1098,17 +1094,8 @@ bool NativeObject::fixupAfterSwap(JSContext* cx, Handle<NativeObject*> obj,
     obj->initSlotUnchecked(i, slotValues[i]);
   }
 
-  if (obj->hasDynamicElements()) {
-    ObjectElements* elements = obj->getElementsHeader();
-    void* allocatedElements = obj->getUnshiftedElementsHeader();
-    MOZ_ASSERT(!cx->nursery().isInside(allocatedElements));
-    size_t size = elements->numAllocatedElements() * sizeof(HeapSlot);
-    if (obj->isTenured()) {
-      AddCellMemory(obj, size, MemoryUse::ObjectElements);
-    } else if (!cx->nursery().registerMallocedBuffer(allocatedElements, size)) {
-      return false;
-    }
-  }
+  MOZ_ASSERT_IF(obj->hasDynamicElements(),
+                gc::IsBufferAlloc(obj->getUnshiftedElementsHeader()));
 
   return true;
 }
@@ -1299,7 +1286,6 @@ void JSObject::swap(JSContext* cx, HandleObject a, HandleObject b,
     js_memcpy(a, b, size);
     js_memcpy(b, tmp, size);
 
-    zone->swapCellMemory(a, b, MemoryUse::ObjectElements);
     zone->swapCellMemory(a, b, MemoryUse::ProxyExternalValueArray);
 
     if (aIsProxyWithInlineValues) {
@@ -3319,14 +3305,16 @@ js::gc::AllocKind JSObject::allocKindForTenure(
 void JSObject::addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf,
                                       JS::ClassInfo* info,
                                       JS::RuntimeSizes* runtimeSizes) {
+  // TODO: These will eventually count as GC heap memory.
   if (is<NativeObject>() && as<NativeObject>().hasDynamicSlots()) {
     info->objectsMallocHeapSlots +=
         gc::GetAllocSize(as<NativeObject>().getSlotsHeader());
   }
 
   if (is<NativeObject>() && as<NativeObject>().hasDynamicElements()) {
     void* allocatedElements = as<NativeObject>().getUnshiftedElementsHeader();
-    info->objectsMallocHeapElementsNormal += mallocSizeOf(allocatedElements);
+    info->objectsMallocHeapElementsNormal +=
+        gc::GetAllocSize(allocatedElements);
   }
 
   // Other things may be measured in the future if DMD indicates it is
@@ -3448,6 +3436,11 @@ void JSObject::traceChildren(JSTracer* trc) {
       }
     }
 
+    if (nobj->hasDynamicElements()) {
+      TraceEdgeToBuffer(trc, nobj, nobj->getUnshiftedElementsHeader(),
+                        "objectDynamicElements allocation");
+    }
+
     TraceRange(trc, nobj->getDenseInitializedLength(),
                nobj->getDenseElements().begin(), "objectElements");
   }

js/src/vm/NativeObject.cpp

@@ -806,7 +806,7 @@ bool NativeObject::goodElementsAllocationAmount(JSContext* cx,
   const uint32_t Mebi = 1 << 20;
   if (reqAllocated < Mebi) {
     uint32_t amount =
-        mozilla::AssertedCast<uint32_t>(RoundUpPow2(reqAllocated));
+        gc::GetGoodPower2ElementCount(reqAllocated, sizeof(Value));
 
     // If |amount| would be 2/3 or more of the array's length, adjust
     // it (up or down) to be equal to the array's length.  This avoids
@@ -816,11 +816,18 @@ bool NativeObject::goodElementsAllocationAmount(JSContext* cx,
     // opposed to the usual doubling.
     uint32_t goodCapacity = amount - ObjectElements::VALUES_PER_HEADER;
     if (length >= reqCapacity && goodCapacity > (length / 3) * 2) {
-      amount = length + ObjectElements::VALUES_PER_HEADER;
+      amount = gc::GetGoodElementCount(
+          length + ObjectElements::VALUES_PER_HEADER, sizeof(Value));
     }
 
-    if (amount < ELEMENT_CAPACITY_MIN) {
-      amount = ELEMENT_CAPACITY_MIN;
+    const size_t AmountMin =
+        ELEMENT_CAPACITY_MIN + ObjectElements::VALUES_PER_HEADER;
+
+    // Check this size doesn't waste any space in the allocation.
+    MOZ_ASSERT(AmountMin == gc::GetGoodElementCount(AmountMin, sizeof(Value)));
+
+    if (amount < AmountMin) {
+      amount = AmountMin;
     }
 
     *goodAmount = amount;
@@ -857,7 +864,7 @@ bool NativeObject::goodElementsAllocationAmount(JSContext* cx,
   // Pick the first bucket that'll fit |reqAllocated|.
   for (uint32_t b : BigBuckets) {
     if (b >= reqAllocated) {
-      *goodAmount = b;
+      *goodAmount = gc::GetGoodElementCount(b, sizeof(Value));
       return true;
     }
   }
@@ -961,8 +968,8 @@ bool NativeObject::growElements(JSContext* cx, uint32_t reqCapacity) {
     oldAllocated = oldCapacity + ObjectElements::VALUES_PER_HEADER + numShifted;
 
     // Finally, try to resize the buffer.
-    newHeaderSlots = ReallocNurseryOrMallocBuffer<HeapSlot>(
-        cx, this, oldHeaderSlots, oldAllocated, newAllocated, js::MallocArena);
+    newHeaderSlots = ReallocateCellBuffer<HeapSlot>(cx, this, oldHeaderSlots,
+                                                    oldAllocated, newAllocated);
     if (!newHeaderSlots) {
       return false;  // If the resizing failed, then we leave elements at its
                      // old size.
@@ -971,8 +978,7 @@ bool NativeObject::growElements(JSContext* cx, uint32_t reqCapacity) {
     // If the object has fixed elements, then we always need to allocate a new
     // buffer, because if we've reached this code, then the requested capacity
     // is greater than the existing inline space available within the object
-    newHeaderSlots =
-        AllocNurseryOrMallocBuffer<HeapSlot>(cx, this, newAllocated);
+    newHeaderSlots = AllocateCellBuffer<HeapSlot>(cx, this, newAllocated);
     if (!newHeaderSlots) {
       return false;  // Leave elements at its old size.
     }
@@ -982,13 +988,6 @@ bool NativeObject::growElements(JSContext* cx, uint32_t reqCapacity) {
             ObjectElements::VALUES_PER_HEADER + initlen + numShifted);
   }
 
-  // If the object already had dynamic elements, then we have to account
-  // for freeing the old elements buffer.
-  if (oldAllocated) {
-    RemoveCellMemory(this, oldAllocated * sizeof(HeapSlot),
-                     MemoryUse::ObjectElements);
-  }
-
   ObjectElements* newheader = reinterpret_cast<ObjectElements*>(newHeaderSlots);
   // Update the elements pointer to point to the new elements buffer.
   elements_ = newheader->elements() + numShifted;
@@ -1001,10 +1000,6 @@ bool NativeObject::growElements(JSContext* cx, uint32_t reqCapacity) {
   // Poison the uninitialized portion of the new elements buffer.
   Debug_SetSlotRangeToCrashOnTouch(elements_ + initlen, newCapacity - initlen);
 
-  // Account for allocating the new elements buffer.
-  AddCellMemory(this, newAllocated * sizeof(HeapSlot),
-                MemoryUse::ObjectElements);
-
   return true;
 }
 
@@ -1044,22 +1039,16 @@ void NativeObject::shrinkElements(JSContext* cx, uint32_t reqCapacity) {
 
   HeapSlot* oldHeaderSlots =
       reinterpret_cast<HeapSlot*>(getUnshiftedElementsHeader());
-  HeapSlot* newHeaderSlots = ReallocNurseryOrMallocBuffer<HeapSlot>(
-      cx, this, oldHeaderSlots, oldAllocated, newAllocated, js::MallocArena);
+  HeapSlot* newHeaderSlots = ReallocateCellBuffer<HeapSlot>(
+      cx, this, oldHeaderSlots, oldAllocated, newAllocated);
   if (!newHeaderSlots) {
     cx->recoverFromOutOfMemory();
     return;  // Leave elements at its old size.
   }
 
-  RemoveCellMemory(this, oldAllocated * sizeof(HeapSlot),
-                   MemoryUse::ObjectElements);
-
   ObjectElements* newheader = reinterpret_cast<ObjectElements*>(newHeaderSlots);
   elements_ = newheader->elements() + numShifted;
   getElementsHeader()->capacity = newCapacity;
-
-  AddCellMemory(this, newAllocated * sizeof(HeapSlot),
-                MemoryUse::ObjectElements);
 }
 
 void NativeObject::shrinkCapacityToInitializedLength(JSContext* cx) {
@@ -1084,19 +1073,7 @@ void NativeObject::shrinkCapacityToInitializedLength(JSContext* cx) {
 
   shrinkElements(cx, len);
 
-  header = getElementsHeader();
-  uint32_t oldAllocated = header->numAllocatedElements();
-  header->capacity = len;
-
-  // The size of the memory allocation hasn't changed but we lose the actual
-  // capacity information. Make the associated size match the updated capacity.
-  if (!hasFixedElements()) {
-    uint32_t newAllocated = header->numAllocatedElements();
-    RemoveCellMemory(this, oldAllocated * sizeof(HeapSlot),
-                     MemoryUse::ObjectElements);
-    AddCellMemory(this, newAllocated * sizeof(HeapSlot),
-                  MemoryUse::ObjectElements);
-  }
+  getElementsHeader()->capacity = len;
 }
 
 /* static */

js/src/vm/NativeObject.h

@@ -881,8 +881,7 @@ class NativeObject : public JSObject {
   /*
    * Minimum size for dynamically allocated elements in normal Objects.
    */
-  static const uint32_t ELEMENT_CAPACITY_MIN =
-      8 - ObjectElements::VALUES_PER_HEADER;
+  static const uint32_t ELEMENT_CAPACITY_MIN = 5;
 
   HeapSlot* fixedSlots() const {
     return reinterpret_cast<HeapSlot*>(uintptr_t(this) + sizeof(NativeObject));