Feature/prod 4452/add cache thrash detector

HzMemoryCache/Diagnostics/ThrashingDetector.cs

@@ -0,0 +1,8 @@
+namespace HzCache.Diagnostics
+{
+    internal class ThrashingDetector(string checksum)
+    {
+        public int Counter { get; set; } = 0;
+        public string Checksum => checksum;
+    }
+}

HzMemoryCache/HzMemoryCache.cs

@@ -1,4 +1,9 @@
 #nullable enable
+
+using HzCache.Diagnostics;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using System;
 using System.Collections;
 using System.Collections.Concurrent;
@@ -10,8 +15,6 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
-using HzCache.Diagnostics;
-using Microsoft.Extensions.Logging;
 
 namespace HzCache
 {
@@ -27,6 +30,7 @@ public partial class HzMemoryCache : IEnumerable<KeyValuePair<string, object>>,
         private readonly ConcurrentDictionary<string, TTLValue> dictionary = new();
         private readonly HzCacheMemoryLocker memoryLocker = new(new HzCacheMemoryLockerOptions());
         private readonly HzCacheOptions options;
+        private readonly MemoryCache thrashingDetectorCache = new(new MemoryCacheOptions());
 
         //IDispisable members
         private bool _disposedValue;
@@ -244,7 +248,7 @@ public bool Remove(string key, bool sendBackplaneNotification, Func<string, bool
 
         public CacheStatistics GetStatistics()
         {
-            return new CacheStatistics {Counts = Count, SizeInBytes = SizeInBytes};
+            return new CacheStatistics { Counts = Count, SizeInBytes = SizeInBytes };
         }
 
         /// <summary>
@@ -292,7 +296,7 @@ public static IPropagatorBlock<TIn, IList<TIn>> CreateBuffer<TIn>(TimeSpan timeS
 
         private void StartUpdateChecksumAndNotify()
         {
-            var options = new DataflowLinkOptions {PropagateCompletion = true};
+            var options = new DataflowLinkOptions { PropagateCompletion = true };
             var actionBlock = new ActionBlock<IList<TTLValue>>(ttlValues =>
             {
                 try
@@ -325,6 +329,8 @@ private bool RemoveItem(string key, CacheItemChangeType changeType, bool sendNot
 
             if (result)
             {
+                DetectCacheThrashing(key, ttlValue?.checksum);
+
                 result = dictionary.TryRemove(key, out ttlValue);
                 if (result)
                 {
@@ -340,6 +346,48 @@ private bool RemoveItem(string key, CacheItemChangeType changeType, bool sendNot
             return result;
         }
 
+        // Remember the value we are removing from the local cache.
+        // If the same value is being removed again and again in a short time frame,
+        // we are likely experiencing cache thrashing.
+        private void DetectCacheThrashing(string key, string? ttlValueChecksum)
+        {
+            try
+            {
+                if (!options.LogCacheThrashing)
+                {
+                    return;
+                }
+
+                if (ttlValueChecksum == null || options.logger == null)
+                    return;
+                if (!thrashingDetectorCache.TryGetValue(key, out ThrashingDetector? thrashingDetector))
+                {
+                    thrashingDetectorCache.Set(key, new ThrashingDetector(ttlValueChecksum), options.ThrashingWindow);
+                    return;
+                }
+
+                if (thrashingDetector.Checksum == ttlValueChecksum)
+                {
+                    thrashingDetector.Counter++;
+                }
+                else
+                {
+                    thrashingDetectorCache.Remove(key);
+                }
+
+                if (thrashingDetector.Counter == options.ThrashingLimit)
+                {
+                    options.logger.LogWarning(
+                        "Cache Thrashing Detected: {Key} has been removed from local cache {ThrashingLimit} times in the last {ThrashingWindow}s. Checksum of existing value:{Checksum}",
+                        key, options.ThrashingLimit, options.ThrashingWindow.TotalSeconds, ttlValueChecksum);
+                }
+            }
+            catch (Exception e)
+            {
+                options.logger?.LogError(e, "Error in DetectCacheThrashing {Key}", key);
+            }
+        }
+
         private void NotifyItemChange(string key, CacheItemChangeType changeType, TTLValue ttlValue, byte[]? objectData = null, bool isPattern = false)
         {
             options.valueChangeListener(key, changeType, ttlValue, objectData, isPattern);
@@ -357,6 +405,7 @@ private void Dispose(bool disposing)
                 if (disposing)
                 {
                     cleanUpTimer.Dispose();
+                    thrashingDetectorCache.Dispose();
                 }
 
                 _disposedValue = true;
@@ -405,4 +454,4 @@ public static bool IsNullOrDefault<T>(T argument)
             return false;
         }
     }
-}
+}

HzMemoryCache/IHzCache.cs

@@ -1,4 +1,5 @@
 #nullable enable
+
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
@@ -29,7 +30,6 @@ public enum NotificationType
         Async, Sync, None
     }
 
-
     /// <summary>
     ///     The eviction policy to use for the cache.
     ///     LRU is "Least Recently Used" and FIFO is "First In First Out". Which is almost true.
@@ -92,6 +92,21 @@ public class HzCacheOptions
         /// benefit of compression.
         /// </summary>
         public long compressionThreshold { get; set; } = Int64.MaxValue;
+
+        /// <summary>
+        ///     Enables logging of cache thrashing events, which occur when keys are removed frequently.
+        /// </summary>
+        public bool LogCacheThrashing { get; set; } = false;
+
+        /// <summary>
+        ///     The time window during which cache removals are counted for thrashing detection.
+        /// </summary>
+        public TimeSpan ThrashingWindow { get; set; } = TimeSpan.FromSeconds(60);
+
+        /// <summary>
+        ///     The number of removals within <see cref="ThrashingWindow"/> that triggers a cache thrashing warning.
+        /// </summary>
+        public int ThrashingLimit { get; set; } = 3;
     }
 
     public interface IHzCache
@@ -182,6 +197,7 @@ public interface IHzCache
         bool Remove(string key);
 
         Task ClearAsync();
+
         Task<bool> RemoveAsync(string key);
     }
 
@@ -198,7 +214,6 @@ public interface IDetailedHzCache : IHzCache
         /// </summary>
         void Clear();
 
-
         /// <summary>
         ///     Tries to remove item with the specified key, also returns the object removed in an "out" var
         /// </summary>

RedisBackedHzCache/RedisBackedHzCache.cs

@@ -97,7 +97,10 @@ public RedisBackedHzCache(RedisBackedHzCacheOptions options)
                         }
                     }
                 },
-                defaultTTL = options.defaultTTL
+                defaultTTL = options.defaultTTL,
+                LogCacheThrashing = options.LogCacheThrashing,
+                ThrashingLimit = options.ThrashingLimit,
+                ThrashingWindow = options.ThrashingWindow,
             });
 
             if (string.IsNullOrWhiteSpace(options.redisConnectionString))
@@ -181,7 +184,7 @@ public CacheStatistics GetStatistics()
 
         public T Get<T>(string key)
         {
-            using var activity = HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.Get, HzActivities.Area.RedisBackedHzCache, key: key); 
+            using var activity = HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.Get, HzActivities.Area.RedisBackedHzCache, key: key);
             var value = hzCache.Get<T>(key);
             if (value == null && options.useRedisAs2ndLevelCache)
             {
@@ -226,7 +229,7 @@ public IList<T> GetOrSetBatch<T>(IList<string> keys, Func<IList<string>, List<Ke
 
         public IList<T> GetOrSetBatch<T>(IList<string> keys, Func<IList<string>, List<KeyValuePair<string, T>>> valueFactory, TimeSpan ttl)
         {
-            using var activity = HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.GetOrSetBatch, HzActivities.Area.RedisBackedHzCache, key: string.Join(",",keys??new List<string>()));
+            using var activity = HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.GetOrSetBatch, HzActivities.Area.RedisBackedHzCache, key: string.Join(",", keys ?? new List<string>()));
             Func<IList<string>, List<KeyValuePair<string, T>>> redisFactory = idList =>
             {
                 // Create a list of redis keys from the list of cache keys
@@ -290,4 +293,4 @@ private string CacheKeyFromRedisKey(string redisKey)
             return redisKey.Substring(options.applicationCachePrefix.Length + 1);
         }
     }
-}
+}