Feature/prod 4452/add cache thrash detector

HzMemoryCache/Diagnostics/TrashingDetector.cs

@@ -0,0 +1,8 @@
+namespace HzCache.Diagnostics
+{
+    internal class TrashingDetector
+    {
+        public string Checksum { get; set; }
+        public int Counter { get; set; }
+    }
+}

HzMemoryCache/HzMemoryCache.cs

@@ -1,4 +1,9 @@
 #nullable enable
+
+using HzCache.Diagnostics;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using System;
 using System.Collections;
 using System.Collections.Concurrent;
@@ -10,8 +15,6 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
-using HzCache.Diagnostics;
-using Microsoft.Extensions.Logging;
 
 namespace HzCache
 {
@@ -27,6 +30,7 @@ public partial class HzMemoryCache : IEnumerable<KeyValuePair<string, object>>,
         private readonly ConcurrentDictionary<string, TTLValue> dictionary = new();
         private readonly HzCacheMemoryLocker memoryLocker = new(new HzCacheMemoryLockerOptions());
         private readonly HzCacheOptions options;
+        private readonly MemoryCache trashingDetectorCache = new(new MemoryCacheOptions());
 
         //IDispisable members
         private bool _disposedValue;
@@ -244,7 +248,7 @@ public bool Remove(string key, bool sendBackplaneNotification, Func<string, bool
 
         public CacheStatistics GetStatistics()
         {
-            return new CacheStatistics {Counts = Count, SizeInBytes = SizeInBytes};
+            return new CacheStatistics { Counts = Count, SizeInBytes = SizeInBytes };
         }
 
         /// <summary>
@@ -292,7 +296,7 @@ public static IPropagatorBlock<TIn, IList<TIn>> CreateBuffer<TIn>(TimeSpan timeS
 
         private void StartUpdateChecksumAndNotify()
         {
-            var options = new DataflowLinkOptions {PropagateCompletion = true};
+            var options = new DataflowLinkOptions { PropagateCompletion = true };
             var actionBlock = new ActionBlock<IList<TTLValue>>(ttlValues =>
             {
                 try
@@ -325,6 +329,8 @@ private bool RemoveItem(string key, CacheItemChangeType changeType, bool sendNot
 
             if (result)
             {
+                DetectCacheTrashing(key, ttlValue?.checksum);
+
                 result = dictionary.TryRemove(key, out ttlValue);
                 if (result)
                 {
@@ -340,6 +346,51 @@ private bool RemoveItem(string key, CacheItemChangeType changeType, bool sendNot
             return result;
         }
 
+        //Remember the value we are removing from the local cache, if the same value is being removed again and again in a short time frame, we are likely experiencing cache trashing.
+        private void DetectCacheTrashing(string key, string ttlValueChecksum)
+        {
+            try
+            {
+                if (!options.LogCacheTrashing)
+                {
+                    return;
+                }
+
+                if (ttlValueChecksum == null || options.logger == null)
+                    return;
+                TrashingDetector? trashingDetector;
+                if (!trashingDetectorCache.TryGetValue(key, out trashingDetector))
+                {
+                    trashingDetectorCache.Set(key, new TrashingDetector
+                    {
+                        Checksum = ttlValueChecksum,
+                        Counter = 0
+                    }, options.TrashingWindow);
+                    return;
+                }
+
+                if (trashingDetector.Checksum == ttlValueChecksum)
+                {
+                    trashingDetector.Counter++;
+                }
+                else
+                {
+                    trashingDetectorCache.Remove(key);
+                }
+
+                if (trashingDetector.Counter >= options.TrashingLimit)
+                {
+                    options.logger.LogWarning(
+                        "Cache Trashing Detected: {Key} has been removed from local cache 3 times last 60s. Checksum of existing value:{Checksum}",
+                        key, ttlValueChecksum);
+                }
+            }
+            catch (Exception e)
+            {
+                options.logger?.LogError(e, "Error in DetectCacheTrashing {Key}", key);
+            }
+        }
+
         private void NotifyItemChange(string key, CacheItemChangeType changeType, TTLValue ttlValue, byte[]? objectData = null, bool isPattern = false)
         {
             options.valueChangeListener(key, changeType, ttlValue, objectData, isPattern);
@@ -357,6 +408,7 @@ private void Dispose(bool disposing)
                 if (disposing)
                 {
                     cleanUpTimer.Dispose();
+                    trashingDetectorCache.Dispose();
                 }
 
                 _disposedValue = true;
@@ -405,4 +457,4 @@ public static bool IsNullOrDefault<T>(T argument)
             return false;
         }
     }
-}
+}

HzMemoryCache/IHzCache.cs

@@ -1,4 +1,5 @@
 #nullable enable
+
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
@@ -29,7 +30,6 @@ public enum NotificationType
         Async, Sync, None
     }
 
-
     /// <summary>
     ///     The eviction policy to use for the cache.
     ///     LRU is "Least Recently Used" and FIFO is "First In First Out". Which is almost true.
@@ -92,6 +92,10 @@ public class HzCacheOptions
         /// benefit of compression.
         /// </summary>
         public long compressionThreshold { get; set; } = Int64.MaxValue;
+
+        public bool LogCacheTrashing { get; set; } = false;
+        public TimeSpan TrashingWindow { get; set; } = TimeSpan.FromSeconds(60);
+        public int TrashingLimit { get; set; } = 3;
     }
 
     public interface IHzCache
@@ -182,6 +186,7 @@ public interface IHzCache
         bool Remove(string key);
 
         Task ClearAsync();
+
         Task<bool> RemoveAsync(string key);
     }
 
@@ -198,7 +203,6 @@ public interface IDetailedHzCache : IHzCache
         /// </summary>
         void Clear();
 
-
         /// <summary>
         ///     Tries to remove item with the specified key, also returns the object removed in an "out" var
         /// </summary>