Feature/prod 4452/add cache thrash detector

HzMemoryCache/HzMemoryCache.cs

@@ -1,4 +1,8 @@
 #nullable enable
+
+using HzCache.Diagnostics;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
 using System;
 using System.Collections;
 using System.Collections.Concurrent;
@@ -10,8 +14,6 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
-using HzCache.Diagnostics;
-using Microsoft.Extensions.Logging;
 
 namespace HzCache
 {
@@ -27,6 +29,7 @@ public partial class HzMemoryCache : IEnumerable<KeyValuePair<string, object>>,
         private readonly ConcurrentDictionary<string, TTLValue> dictionary = new();
         private readonly HzCacheMemoryLocker memoryLocker = new(new HzCacheMemoryLockerOptions());
         private readonly HzCacheOptions options;
+        private readonly MemoryCache trashDetectorCache = new(new MemoryCacheOptions());
 
         //IDispisable members
         private bool _disposedValue;
@@ -244,7 +247,7 @@ public bool Remove(string key, bool sendBackplaneNotification, Func<string, bool
 
         public CacheStatistics GetStatistics()
         {
-            return new CacheStatistics {Counts = Count, SizeInBytes = SizeInBytes};
+            return new CacheStatistics { Counts = Count, SizeInBytes = SizeInBytes };
         }
 
         /// <summary>
@@ -292,7 +295,7 @@ public static IPropagatorBlock<TIn, IList<TIn>> CreateBuffer<TIn>(TimeSpan timeS
 
         private void StartUpdateChecksumAndNotify()
         {
-            var options = new DataflowLinkOptions {PropagateCompletion = true};
+            var options = new DataflowLinkOptions { PropagateCompletion = true };
             var actionBlock = new ActionBlock<IList<TTLValue>>(ttlValues =>
             {
                 try
@@ -325,6 +328,8 @@ private bool RemoveItem(string key, CacheItemChangeType changeType, bool sendNot
 
             if (result)
             {
+                DetectCacheTrashing(key, ttlValue?.checksum);
+
                 result = dictionary.TryRemove(key, out ttlValue);
                 if (result)
                 {
@@ -340,6 +345,46 @@ private bool RemoveItem(string key, CacheItemChangeType changeType, bool sendNot
             return result;
         }
 
+        //Remember the value we are removing from the local cache, if the same value is being removed again and again in a short time frame, we are likely experiencing cache trashing.
+        private void DetectCacheTrashing(string key, string ttlValueChecksum)
+        {
+            try
+            {
+                if (ttlValueChecksum == null || options.logger == null)
+                    return;
+                TrashDetector trashDetector;
+                if (!trashDetectorCache.TryGetValue(key, out trashDetector))
+                {
+                    trashDetectorCache.Set(key, new TrashDetector
+                    {
+                        Checksum = ttlValueChecksum,
+                        Counter = 0
+                    }, TimeSpan.FromSeconds(60));
+                    return;
+                }
+
+                if (trashDetector.Checksum == ttlValueChecksum)
+                {
+                    trashDetector.Counter++;
+                }
+                else
+                {
+                    trashDetectorCache.Remove(key);
+                }
+
+                if (trashDetector.Counter == 3)
+                {
+                    options.logger.LogWarning(
+                        $"Cache Trashing Detected: {key} has been removed from local cache 3 times last 60s. Checksum of existing value:{ttlValueChecksum}",
+                        key, ttlValueChecksum);
+                }
+            }
+            catch (Exception e)
+            {
+                options.logger?.LogError(e, $"Error in DetectCacheTrashing {key}", key);
+            }
+        }
+
         private void NotifyItemChange(string key, CacheItemChangeType changeType, TTLValue ttlValue, byte[]? objectData = null, bool isPattern = false)
         {
             options.valueChangeListener(key, changeType, ttlValue, objectData, isPattern);
@@ -405,4 +450,10 @@ public static bool IsNullOrDefault<T>(T argument)
             return false;
         }
     }
-}
+
+    public class TrashDetector
+    {
+        public string Checksum { get; set; }
+        public int Counter { get; set; }
+    }
+}