Merge branch 'main' into users/alexpeck/avx2

bitfaster · web-flow · commit 23584fe67f44 · 2024-06-09T15:45:42.000-07:00
diff --git a/BitFaster.Caching.ThroughputAnalysis/SizeExec.sh b/BitFaster.Caching.ThroughputAnalysis/SizeExec.sh
@@ -0,0 +1,21 @@
+cls
+
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 1 100
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 1 10000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 1 1000000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 1 10000000
+
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 2 1000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 2 100000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 2 10000000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 2 100000000
+
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 3 100
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 3 10000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 3 1000000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 3 10000000
+
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 4 100
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 4 10000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 4 1000000
+dotnet BitFaster.Caching.ThroughputAnalysis.dll 4 10000000
diff --git a/BitFaster.Caching.ThroughputAnalysis/ThroughputBenchmark.cs b/BitFaster.Caching.ThroughputAnalysis/ThroughputBenchmark.cs
@@ -31,7 +31,7 @@ public abstract class ThroughputBenchmarkBase
             Initialize?.Invoke(cache);
 
             // Warmup a few times before estimating run time
-            config.Iterations = 10;
+            config.Iterations = 1;
 
             for (int i = 0; i < warmup; i++)
             {
@@ -46,14 +46,17 @@ public abstract class ThroughputBenchmarkBase
                 var sw = Stopwatch.StartNew();
                 Run(Stage.Pilot, 0, threads, config, cache);
 
-                valid = sw.Elapsed > TimeSpan.FromMilliseconds(400) ? valid + 1 : 0;    
+                valid = sw.Elapsed > TimeSpan.FromMilliseconds(800) ? valid + 1 : 0;    
 
                 if (valid > 3)
                 {
                     break;
                 }
 
-                config.Iterations = (int)(1.2 * config.Iterations);
+                if (valid == 0)
+                { 
+                    config.Iterations = config.Iterations < 5 ? config.Iterations + 1 : (int)(1.2 * config.Iterations); 
+                }
             }
 
             int runCounter = 0;
diff --git a/BitFaster.Caching.UnitTests/Lfu/ConcurrentTLfuTests.cs b/BitFaster.Caching.UnitTests/Lfu/ConcurrentTLfuTests.cs
@@ -1,5 +1,6 @@
 ﻿using System;
 using System.Runtime.InteropServices;
+using System.Threading;
 using BitFaster.Caching.Lfu;
 using BitFaster.Caching.Scheduler;
 using BitFaster.Caching.UnitTests.Retry;
@@ -25,6 +26,36 @@ public ConcurrentTLfuTests()
             lfu = new ConcurrentTLfu<int, string>(capacity, new ExpireAfterWrite<int, string>(timeToLive));
         }
 
+        // This is a scenario test to verify maintenance is run promptly after read.
+        [RetryFact]
+        public void WhenItemIsAccessedTimeToExpireIsUpdated()
+        { 
+            var cache = new ConcurrentLfuBuilder<int, int>()
+                .WithCapacity(10)
+                .WithExpireAfterAccess(TimeSpan.FromSeconds(5))
+                .Build();
+
+            Timed.Execute(
+                cache,
+                cache =>
+                {
+                    cache.AddOrUpdate(1, 1);
+                    return cache;
+                },
+                TimeSpan.FromSeconds(4),
+                cache =>
+                {
+                    cache.TryGet(1, out var value);
+                },
+                TimeSpan.FromSeconds(2),
+                cache =>
+                { 
+                    cache.TryGet(1, out var value).Should().BeTrue();
+                    cache.TryGet(1, out value).Should().BeTrue();
+                }
+            );
+        }
+
         [Fact]
         public void ConstructAddAndRetrieveWithCustomComparerReturnsValue()
         {
diff --git a/BitFaster.Caching/BitFaster.Caching.csproj b/BitFaster.Caching/BitFaster.Caching.csproj
@@ -10,7 +10,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <PackageReadmeFile>ReadMe.md</PackageReadmeFile>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
-    <Version>2.5.0</Version>
+    <Version>2.5.1</Version>
     <Copyright>Copyright © Alex Peck $([System.DateTime]::Now.ToString(yyyy))</Copyright>
     <PackageProjectUrl></PackageProjectUrl>
     <RepositoryUrl>https://github.com/bitfaster/BitFaster.Caching</RepositoryUrl>
@@ -22,8 +22,8 @@
     <IncludeSymbols>True</IncludeSymbols>
     <EmbedUntrackedSources>true</EmbedUntrackedSources>
     <SymbolPackageFormat>snupkg</SymbolPackageFormat>
-    <AssemblyVersion>2.5.0.0</AssemblyVersion>
-    <FileVersion>2.5.0.0</FileVersion>
+    <AssemblyVersion>2.5.1.0</AssemblyVersion>
+    <FileVersion>2.5.1.0</FileVersion>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
     <IsTrimmable>true</IsTrimmable>
diff --git a/BitFaster.Caching/Lfu/ConcurrentLfuCore.cs b/BitFaster.Caching/Lfu/ConcurrentLfuCore.cs
@@ -276,6 +276,7 @@ private bool TryGetImpl(K key, [MaybeNullWhen(false)] out V value)
                     {
                         TryScheduleDrain();
                     }
+                    this.policy.OnRead(node);
                     value = node.Value;
                     return true;
                 }
@@ -366,6 +367,7 @@ public bool TryUpdate(K key, V value)
                 // and we will just lose ordering/hit count, but not orphan the node.
                 this.writeBuffer.TryAdd(node);
                 TryScheduleDrain();
+                this.policy.OnWrite(node);
                 return true;
             }
 
@@ -525,8 +527,6 @@ private bool Maintenance(N? droppedWrite = null)
         {
             this.drainStatus.VolatileWrite(DrainStatus.ProcessingToIdle);
 
-            policy.AdvanceTime();
-
             // Note: this is only Span on .NET Core 3.1+, else this is no-op and it is still an array
             var buffer = this.drainBuffer.AsSpanOrArray();
 
@@ -601,7 +601,7 @@ private void OnAccess(N node)
                     break;
             }
 
-            policy.OnRead(node);
+            policy.AfterRead(node);
         }
 
         private void OnWrite(N node)
@@ -650,7 +650,7 @@ private void OnWrite(N node)
                     break;
             }
 
-            policy.OnWrite(node);
+            policy.AfterWrite(node);
         }
 
         private void PromoteProbation(LfuNode<K, V> node)
diff --git a/BitFaster.Caching/Lfu/NodePolicy.cs b/BitFaster.Caching/Lfu/NodePolicy.cs
@@ -10,9 +10,10 @@ internal interface INodePolicy<K, V, N>
     {
         N Create(K key, V value);
         bool IsExpired(N node);
-        void AdvanceTime();
         void OnRead(N node);
         void OnWrite(N node);
+        void AfterRead(N node);
+        void AfterWrite(N node);
         void OnEvict(N node);
         void ExpireEntries<P>(ref ConcurrentLfuCore<K, V, N, P> cache) where P : struct, INodePolicy<K, V, N>;
     }
@@ -33,17 +34,22 @@ public bool IsExpired(AccessOrderNode<K, V> node)
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void AdvanceTime()
+        public void OnRead(AccessOrderNode<K, V> node)
         {
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void OnRead(AccessOrderNode<K, V> node)
+        public void OnWrite(AccessOrderNode<K, V> node)
         {
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void OnWrite(AccessOrderNode<K, V> node)
+        public void AfterRead(AccessOrderNode<K, V> node)
+        {
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void AfterWrite(AccessOrderNode<K, V> node)
         {
         }
 
@@ -84,29 +90,33 @@ public TimeOrderNode<K, V> Create(K key, V value)
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool IsExpired(TimeOrderNode<K, V> node)
-        {
-            return node.TimeToExpire < Duration.SinceEpoch();
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void AdvanceTime()
         {
             current = Duration.SinceEpoch();
+            return node.TimeToExpire < current;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void OnRead(TimeOrderNode<K, V> node)
         {
-            var oldTte = node.TimeToExpire;
+            // we know IsExpired is always called immediate before OnRead, so piggyback on the current time
             node.TimeToExpire = current + expiryCalculator.GetExpireAfterRead(node.Key, node.Value, node.TimeToExpire - current);
-            if (oldTte.raw != node.TimeToExpire.raw)
-            {
-                wheel.Reschedule(node);
-            }
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void OnWrite(TimeOrderNode<K, V> node)
+        {
+            var current = Duration.SinceEpoch();
+            node.TimeToExpire = current + expiryCalculator.GetExpireAfterUpdate(node.Key, node.Value, node.TimeToExpire - current);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void AfterRead(TimeOrderNode<K, V> node)
+        {
+            wheel.Reschedule(node);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void AfterWrite(TimeOrderNode<K, V> node)
         {
             // if the node is not yet scheduled, it is being created
             // the time is set on create in case it is read before the buffer is processed
@@ -116,12 +126,7 @@ public void OnWrite(TimeOrderNode<K, V> node)
             }
             else
             {
-                var oldTte = node.TimeToExpire;
-                node.TimeToExpire = current + expiryCalculator.GetExpireAfterUpdate(node.Key, node.Value, node.TimeToExpire - current);
-                if (oldTte.raw != node.TimeToExpire.raw)
-                {
-                    wheel.Reschedule(node);
-                }
+                wheel.Reschedule(node);
             }
         }
 
@@ -134,7 +139,7 @@ public void OnEvict(TimeOrderNode<K, V> node)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void ExpireEntries<P>(ref ConcurrentLfuCore<K, V, TimeOrderNode<K, V>, P> cache) where P : struct, INodePolicy<K, V, TimeOrderNode<K, V>>
         {
-            wheel.Advance(ref cache, current);
+            wheel.Advance(ref cache, Duration.SinceEpoch());
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ public abstract class ThroughputBenchmarkBase`
`31`	`31`	`Initialize?.Invoke(cache);`
`32`	`32`
`33`	`33`	`// Warmup a few times before estimating run time`
`34`		`- config.Iterations = 10;`
	`34`	`+ config.Iterations = 1;`
`35`	`35`
`36`	`36`	`for (int i = 0; i < warmup; i++)`
`37`	`37`	`{`
`@@ -46,14 +46,17 @@ public abstract class ThroughputBenchmarkBase`
`46`	`46`	`var sw = Stopwatch.StartNew();`
`47`	`47`	`Run(Stage.Pilot, 0, threads, config, cache);`
`48`	`48`
`49`		`- valid = sw.Elapsed > TimeSpan.FromMilliseconds(400) ? valid + 1 : 0;`
	`49`	`+ valid = sw.Elapsed > TimeSpan.FromMilliseconds(800) ? valid + 1 : 0;`
`50`	`50`
`51`	`51`	`if (valid > 3)`
`52`	`52`	`{`
`53`	`53`	`break;`
`54`	`54`	`}`
`55`	`55`
`56`		`- config.Iterations = (int)(1.2 * config.Iterations);`
	`56`	`+ if (valid == 0)`
	`57`	`+ {`
	`58`	`+ config.Iterations = config.Iterations < 5 ? config.Iterations + 1 : (int)(1.2 * config.Iterations);`
	`59`	`+ }`
`57`	`60`	`}`
`58`	`61`
`59`	`62`	`int runCounter = 0;`
Original file line number	Diff line number	Diff line change
`@@ -276,6 +276,7 @@ private bool TryGetImpl(K key, [MaybeNullWhen(false)] out V value)`
`276`	`276`	`{`
`277`	`277`	`TryScheduleDrain();`
`278`	`278`	`}`
	`279`	`+ this.policy.OnRead(node);`
`279`	`280`	`value = node.Value;`
`280`	`281`	`return true;`
`281`	`282`	`}`
`@@ -366,6 +367,7 @@ public bool TryUpdate(K key, V value)`
`366`	`367`	`// and we will just lose ordering/hit count, but not orphan the node.`
`367`	`368`	`this.writeBuffer.TryAdd(node);`
`368`	`369`	`TryScheduleDrain();`
	`370`	`+ this.policy.OnWrite(node);`
`369`	`371`	`return true;`
`370`	`372`	`}`
`371`	`373`
`@@ -525,8 +527,6 @@ private bool Maintenance(N? droppedWrite = null)`
`525`	`527`	`{`
`526`	`528`	`this.drainStatus.VolatileWrite(DrainStatus.ProcessingToIdle);`
`527`	`529`
`528`		`- policy.AdvanceTime();`
`529`		`-`
`530`	`530`	`// Note: this is only Span on .NET Core 3.1+, else this is no-op and it is still an array`
`531`	`531`	`var buffer = this.drainBuffer.AsSpanOrArray();`
`532`	`532`
`@@ -601,7 +601,7 @@ private void OnAccess(N node)`
`601`	`601`	`break;`
`602`	`602`	`}`
`603`	`603`
`604`		`- policy.OnRead(node);`
	`604`	`+ policy.AfterRead(node);`
`605`	`605`	`}`
`606`	`606`
`607`	`607`	`private void OnWrite(N node)`
`@@ -650,7 +650,7 @@ private void OnWrite(N node)`
`650`	`650`	`break;`
`651`	`651`	`}`
`652`	`652`
`653`		`- policy.OnWrite(node);`
	`653`	`+ policy.AfterWrite(node);`
`654`	`654`	`}`
`655`	`655`
`656`	`656`	`private void PromoteProbation(LfuNode<K, V> node)`
Original file line number	Diff line number	Diff line change
`@@ -10,9 +10,10 @@ internal interface INodePolicy<K, V, N>`
`10`	`10`	`{`
`11`	`11`	`N Create(K key, V value);`
`12`	`12`	`bool IsExpired(N node);`
`13`		`- void AdvanceTime();`
`14`	`13`	`void OnRead(N node);`
`15`	`14`	`void OnWrite(N node);`
	`15`	`+ void AfterRead(N node);`
	`16`	`+ void AfterWrite(N node);`
`16`	`17`	`void OnEvict(N node);`
`17`	`18`	`void ExpireEntries<P>(ref ConcurrentLfuCore<K, V, N, P> cache) where P : struct, INodePolicy<K, V, N>;`
`18`	`19`	`}`
`@@ -33,17 +34,22 @@ public bool IsExpired(AccessOrderNode<K, V> node)`
`33`	`34`	`}`
`34`	`35`
`35`	`36`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`36`		`- public void AdvanceTime()`
	`37`	`+ public void OnRead(AccessOrderNode<K, V> node)`
`37`	`38`	`{`
`38`	`39`	`}`
`39`	`40`
`40`	`41`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`41`		`- public void OnRead(AccessOrderNode<K, V> node)`
	`42`	`+ public void OnWrite(AccessOrderNode<K, V> node)`
`42`	`43`	`{`
`43`	`44`	`}`
`44`	`45`
`45`	`46`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`46`		`- public void OnWrite(AccessOrderNode<K, V> node)`
	`47`	`+ public void AfterRead(AccessOrderNode<K, V> node)`
	`48`	`+ {`
	`49`	`+ }`
	`50`	`+`
	`51`	`+ [MethodImpl(MethodImplOptions.AggressiveInlining)]`
	`52`	`+ public void AfterWrite(AccessOrderNode<K, V> node)`
`47`	`53`	`{`
`48`	`54`	`}`
`49`	`55`
`@@ -84,29 +90,33 @@ public TimeOrderNode<K, V> Create(K key, V value)`
`84`	`90`
`85`	`91`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`86`	`92`	`public bool IsExpired(TimeOrderNode<K, V> node)`
`87`		`- {`
`88`		`- return node.TimeToExpire < Duration.SinceEpoch();`
`89`		`- }`
`90`		`-`
`91`		`- [MethodImpl(MethodImplOptions.AggressiveInlining)]`
`92`		`- public void AdvanceTime()`
`93`	`93`	`{`
`94`	`94`	`current = Duration.SinceEpoch();`
	`95`	`+ return node.TimeToExpire < current;`
`95`	`96`	`}`
`96`	`97`
`97`	`98`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`98`	`99`	`public void OnRead(TimeOrderNode<K, V> node)`
`99`	`100`	`{`
`100`		`- var oldTte = node.TimeToExpire;`
	`101`	`+ // we know IsExpired is always called immediate before OnRead, so piggyback on the current time`
`101`	`102`	`node.TimeToExpire = current + expiryCalculator.GetExpireAfterRead(node.Key, node.Value, node.TimeToExpire - current);`
`102`		`- if (oldTte.raw != node.TimeToExpire.raw)`
`103`		`- {`
`104`		`- wheel.Reschedule(node);`
`105`		`- }`
`106`	`103`	`}`
`107`	`104`
`108`	`105`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`109`	`106`	`public void OnWrite(TimeOrderNode<K, V> node)`
	`107`	`+ {`
	`108`	`+ var current = Duration.SinceEpoch();`
	`109`	`+ node.TimeToExpire = current + expiryCalculator.GetExpireAfterUpdate(node.Key, node.Value, node.TimeToExpire - current);`
	`110`	`+ }`
	`111`	`+`
	`112`	`+ [MethodImpl(MethodImplOptions.AggressiveInlining)]`
	`113`	`+ public void AfterRead(TimeOrderNode<K, V> node)`
	`114`	`+ {`
	`115`	`+ wheel.Reschedule(node);`
	`116`	`+ }`
	`117`	`+`
	`118`	`+ [MethodImpl(MethodImplOptions.AggressiveInlining)]`
	`119`	`+ public void AfterWrite(TimeOrderNode<K, V> node)`
`110`	`120`	`{`
`111`	`121`	`// if the node is not yet scheduled, it is being created`
`112`	`122`	`// the time is set on create in case it is read before the buffer is processed`
`@@ -116,12 +126,7 @@ public void OnWrite(TimeOrderNode<K, V> node)`
`116`	`126`	`}`
`117`	`127`	`else`
`118`	`128`	`{`
`119`		`- var oldTte = node.TimeToExpire;`
`120`		`- node.TimeToExpire = current + expiryCalculator.GetExpireAfterUpdate(node.Key, node.Value, node.TimeToExpire - current);`
`121`		`- if (oldTte.raw != node.TimeToExpire.raw)`
`122`		`- {`
`123`		`- wheel.Reschedule(node);`
`124`		`- }`
	`129`	`+ wheel.Reschedule(node);`
`125`	`130`	`}`
`126`	`131`	`}`
`127`	`132`
`@@ -134,7 +139,7 @@ public void OnEvict(TimeOrderNode<K, V> node)`
`134`	`139`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`135`	`140`	`public void ExpireEntries<P>(ref ConcurrentLfuCore<K, V, TimeOrderNode<K, V>, P> cache) where P : struct, INodePolicy<K, V, TimeOrderNode<K, V>>`
`136`	`141`	`{`
`137`		`- wheel.Advance(ref cache, current);`
	`142`	`+ wheel.Advance(ref cache, Duration.SinceEpoch());`
`138`	`143`	`}`
`139`	`144`	`}`
`140`	`145`	`}`