diff --git a/BitFaster.Caching/Lfu/CmSketchCore.cs b/BitFaster.Caching/Lfu/CmSketchCore.cs
index cdd70336..5882c57c 100644
--- a/BitFaster.Caching/Lfu/CmSketchCore.cs
+++ b/BitFaster.Caching/Lfu/CmSketchCore.cs
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
-
+using System.Runtime.CompilerServices;
+
#if !NETSTANDARD2_0
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
@@ -83,6 +84,7 @@ public int EstimateFrequency(T value)
/// Increment the count of the specified value.
///
/// The value.
+ [MethodImpl((MethodImplOptions)512)]
public void Increment(T value)
{
#if NETSTANDARD2_0
@@ -269,6 +271,7 @@ private unsafe int EstimateFrequencyAvx(T value)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private unsafe void IncrementAvx(T value)
{
int blockHash = Spread(comparer.GetHashCode(value));
diff --git a/BitFaster.Caching/Lfu/ConcurrentLfu.cs b/BitFaster.Caching/Lfu/ConcurrentLfu.cs
index 0d47aeee..f543e24a 100644
--- a/BitFaster.Caching/Lfu/ConcurrentLfu.cs
+++ b/BitFaster.Caching/Lfu/ConcurrentLfu.cs
@@ -512,6 +512,7 @@ IEnumerator IEnumerable.GetEnumerator()
return ((ConcurrentLfu)this).GetEnumerator();
}
+ [MethodImpl((MethodImplOptions)512)]
private void TryScheduleDrain()
{
if (this.drainStatus.NonVolatileRead() >= DrainStatus.ProcessingToIdle)
@@ -570,6 +571,7 @@ private void DrainBuffers()
}
}
+ [MethodImpl((MethodImplOptions)512)]
private bool Maintenance(LfuNode droppedWrite = null)
{
this.drainStatus.VolatileWrite(DrainStatus.ProcessingToIdle);
@@ -623,6 +625,7 @@ private bool Maintenance(LfuNode droppedWrite = null)
return done;
}
+ [MethodImpl((MethodImplOptions)512)]
private void OnAccess(LfuNode node)
{
// there was a cache hit even if the item was removed or is not yet added.
@@ -648,6 +651,7 @@ private void OnAccess(LfuNode node)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private void OnWrite(LfuNode node)
{
// Nodes can be removed while they are in the write buffer, in which case they should
@@ -695,6 +699,7 @@ private void OnWrite(LfuNode node)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private void PromoteProbation(LfuNode node)
{
if (node.list == null)
@@ -718,12 +723,14 @@ private void PromoteProbation(LfuNode node)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private void EvictEntries()
{
var candidate = EvictFromWindow();
EvictFromMain(candidate);
}
+ [MethodImpl((MethodImplOptions)512)]
private LfuNode EvictFromWindow()
{
LfuNode first = null;
@@ -768,6 +775,7 @@ public void Next()
}
}
+ [MethodImpl((MethodImplOptions)512)]
private void EvictFromMain(LfuNode candidateNode)
{
var victim = new EvictIterator(this.cmSketch, this.probationLru.First); // victims are LRU position in probation
@@ -827,6 +835,7 @@ private void EvictFromMain(LfuNode candidateNode)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private bool AdmitCandidate(K candidateKey, K victimKey)
{
int victimFreq = this.cmSketch.EstimateFrequency(victimKey);
@@ -838,6 +847,7 @@ private bool AdmitCandidate(K candidateKey, K victimKey)
return candidateFreq > victimFreq;
}
+ [MethodImpl((MethodImplOptions)512)]
private void Evict(LfuNode evictee)
{
this.dictionary.TryRemove(evictee.Key, out var _);
@@ -846,6 +856,7 @@ private void Evict(LfuNode evictee)
this.metrics.evictedCount++;
}
+ [MethodImpl((MethodImplOptions)512)]
private void ReFitProtected()
{
// If hill climbing decreased protected, there may be too many items
diff --git a/BitFaster.Caching/Lru/ConcurrentLruCore.cs b/BitFaster.Caching/Lru/ConcurrentLruCore.cs
index 5ecb8dbb..99a55086 100644
--- a/BitFaster.Caching/Lru/ConcurrentLruCore.cs
+++ b/BitFaster.Caching/Lru/ConcurrentLruCore.cs
@@ -565,6 +565,7 @@ private void TrimWarmOrHot(ItemRemovedReason reason)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private void Cycle(int hotCount)
{
if (isWarm)
@@ -630,6 +631,7 @@ private void CycleDuringWarmup(int hotCount)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private (ItemDestination, int) CycleHot(int hotCount)
{
if (hotCount > this.capacity.Hot)
@@ -657,6 +659,7 @@ private void CycleDuringWarmup(int hotCount)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private (ItemDestination, int) CycleWarm(int count)
{
if (count > this.capacity.Warm)
@@ -695,6 +698,7 @@ private void CycleDuringWarmup(int hotCount)
}
}
+ [MethodImpl((MethodImplOptions)512)]
private (ItemDestination, int) CycleCold(int count)
{
if (count > this.capacity.Cold)