first stab at raw API for redis notifications

src/StackExchange.Redis/ConnectionMultiplexer.ClientSideTracking.cs

@@ -0,0 +1,176 @@
+using Microsoft.Extensions.Logging;
+using System;
+using System.Threading;
+using System.Threading.Channels;
+using System.Threading.Tasks;
+
+namespace StackExchange.Redis;
+
+public partial class ConnectionMultiplexer
+{
+    /// <summary>
+    /// Enable the <a href="https://redis.io/commands/client-tracking/">client tracking</a> feature of redis
+    /// </summary>
+    /// <remarks>see also https://redis.io/docs/manual/client-side-caching/</remarks>
+    /// <param name="keyInvalidated">The callback to be invoked when keys are determined to be invalidated</param>
+    /// <param name="options">Additional flags to influence the behavior of client tracking</param>
+    /// <param name="prefixes">Optionally restricts client-side caching notifications for these connections to a subset of key prefixes; this has performance implications (see the PREFIX option in CLIENT TRACKING)</param>
+    public void EnableServerAssistedClientSideTracking(Func<RedisKey, ValueTask> keyInvalidated, ClientTrackingOptions options = ClientTrackingOptions.None, ReadOnlyMemory<RedisKey> prefixes = default)
+    {
+        if (_clientSideTracking is not null) ThrowOnceOnly();
+        if (!prefixes.IsEmpty && (options & ClientTrackingOptions.Broadcast) == 0) ThrowPrefixNeedsBroadcast();
+        var obj = new ClientSideTrackingState(this, keyInvalidated, options, prefixes);
+        if (Interlocked.CompareExchange(ref _clientSideTracking, obj, null) is not null) ThrowOnceOnly();
+
+        static void ThrowOnceOnly() => throw new InvalidOperationException("The " + nameof(EnableServerAssistedClientSideTracking) + " method can be invoked once-only per multiplexer instance");
+        static void ThrowPrefixNeedsBroadcast() => throw new ArgumentException("Prefixes can only be specified when " + nameof(ClientTrackingOptions) + "." + nameof(ClientTrackingOptions.Broadcast) + " is used", nameof(prefixes));
+    }
+
+    private ClientSideTrackingState? _clientSideTracking;
+    internal ClientSideTrackingState? ClientSideTracking => _clientSideTracking;
+    internal sealed class ClientSideTrackingState
+    {
+        public bool IsAlive { get; private set; }
+        private readonly Func<RedisKey, ValueTask> _keyInvalidated;
+        public ClientTrackingOptions Options { get; }
+        public ReadOnlyMemory<RedisKey> Prefixes { get; }
+
+        private readonly Channel<RedisKey> _notifications;
+        private readonly WeakReference<ConnectionMultiplexer> _multiplexer;
+#if NETCOREAPP3_1_OR_GREATER
+        private readonly Action<RedisKey>? _concurrentCallback;
+#else
+            private readonly WaitCallback? _concurrentCallback;
+#endif
+
+        public ClientSideTrackingState(ConnectionMultiplexer multiplexer, Func<RedisKey, ValueTask> keyInvalidated, ClientTrackingOptions options, ReadOnlyMemory<RedisKey> prefixes)
+        {
+            _keyInvalidated = keyInvalidated;
+            Options = options;
+            Prefixes = prefixes;
+            _notifications = Channel.CreateUnbounded<RedisKey>(ChannelOptions);
+            _ = Task.Run(RunAsync);
+            IsAlive = true;
+            _multiplexer = new(multiplexer);
+
+            if ((options & ClientTrackingOptions.ConcurrentInvalidation) != 0)
+            {
+                _concurrentCallback = OnInvalidate;
+            }
+        }
+
+#if !NETCOREAPP3_1_OR_GREATER
+            private void OnInvalidate(object state) => OnInvalidate((RedisKey)state);
+#endif
+
+        private void OnInvalidate(RedisKey key)
+        {
+            try // not optimized for sync completions
+            {
+                var pending = _keyInvalidated(key);
+                if (pending.IsCompleted)
+                {   // observe result
+                    pending.GetAwaiter().GetResult();
+                }
+                else
+                {
+                    _ = ObserveAsyncInvalidation(pending);
+                }
+            }
+            catch (Exception ex) // handle sync failure (via immediate throw or faulted ValueTask)
+            {
+                OnCallbackError(ex);
+            }
+        }
+
+        private async Task ObserveAsyncInvalidation(ValueTask pending)
+        {
+            try
+            {
+                await pending.ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                OnCallbackError(ex);
+            }
+        }
+
+        private ConnectionMultiplexer? Multiplexer => _multiplexer.TryGetTarget(out var multiplexer) ? multiplexer : null;
+
+
+        private void OnCallbackError(Exception error) => Multiplexer?.Logger?.LogError(error, "Client-side tracking invalidation callback failure");
+
+        private async Task RunAsync()
+        {
+            while (await _notifications.Reader.WaitToReadAsync().ConfigureAwait(false))
+            {
+                while (_notifications.Reader.TryRead(out var key))
+                {
+                    if (_concurrentCallback is not null)
+                    {
+#if NETCOREAPP3_1_OR_GREATER
+                        ThreadPool.QueueUserWorkItem(_concurrentCallback, key, preferLocal: false);
+#else
+                            // eat the box
+                            ThreadPool.QueueUserWorkItem(_concurrentCallback, key);
+#endif
+                    }
+                    else
+                    {
+                        try
+                        {
+                            await _keyInvalidated(key).ConfigureAwait(false);
+                        }
+                        catch (Exception ex)
+                        {
+                            OnCallbackError(ex);
+                        }
+                    }
+                }
+            }
+        }
+
+        public void Write(RedisKey key) => _notifications.Writer.TryWrite(key);
+
+        public void Shutdown()
+        {
+            IsAlive = false;
+            _notifications.Writer.TryComplete(null);
+        }
+
+        private static readonly UnboundedChannelOptions ChannelOptions = new UnboundedChannelOptions { SingleReader = true, SingleWriter = false, AllowSynchronousContinuations = true };
+
+
+    }
+}
+
+/// <summary>
+/// Additional flags to influence the behavior of client tracking
+/// </summary>
+[Flags]
+public enum ClientTrackingOptions
+{
+    /// <summary>
+    /// No additional options
+    /// </summary>
+    None = 0,
+    /// <summary>
+    /// Enable tracking in broadcasting mode. In this mode invalidation messages are reported for all the prefixes specified, regardless of the keys requested by the connection. Instead when the broadcasting mode is not enabled, Redis will track which keys are fetched using read-only commands, and will report invalidation messages only for such keys.
+    /// </summary>
+    /// <remarks>This corresponds to CLIENT TRACKING ... BCAST; using <see cref="Broadcast"/> mode consumes less server memory, at the cost of more invalidation messages (i.e. clients are
+    /// likely to receive invalidation messages for keys that the individual client is not using); this can be partially mitigated by using prefixes</remarks>
+    Broadcast = 1 << 0,
+    /// <summary>
+    /// Send notifications about keys modified by this connection itself.
+    /// </summary>
+    /// <remarks>This corresponds to the <b>inverse</b> of CLIENT TRACKING ... NOLOOP; setting <see cref="NotifyForOwnCommands"/> means that your own writes will cause self-notification; this
+    /// may mean that you discard a locally updated copy of the new value, hence this is disabled by default</remarks>
+    NotifyForOwnCommands = 1 << 1,
+
+    /// <summary>
+    /// Indicates that the callback specified for key invalidation should be invoked concurrently rather than sequentially 
+    /// </summary>
+    ConcurrentInvalidation = 1 << 2,
+
+    // to think about: OPTIN / OPTOUT ? I'm happy to implement on the basis of OPTIN for now, though
+}

src/StackExchange.Redis/ConnectionMultiplexer.Events.cs

@@ -14,6 +14,12 @@ public partial class ConnectionMultiplexer
     internal void OnConnectionFailed(EndPoint endpoint, ConnectionType connectionType, ConnectionFailureType failureType, Exception exception, bool reconfigure, string? physicalName)
     {
         if (_isDisposed) return;
+
+        if (connectionType is ConnectionType.Subscription)
+        {
+            GetServerEndPoint(endpoint, activate: false)?.OnSubscriberFailed();
+        }
+
         var handler = ConnectionFailed;
         if (handler != null)
         {

src/StackExchange.Redis/ConnectionMultiplexer.cs

@@ -1,4 +1,7 @@
-using System;
+using Microsoft.Extensions.Logging;
+using Pipelines.Sockets.Unofficial;
+using StackExchange.Redis.Profiling;
+using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
@@ -10,9 +13,6 @@
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Extensions.Logging;
-using Pipelines.Sockets.Unofficial;
-using StackExchange.Redis.Profiling;
 
 namespace StackExchange.Redis
 {
@@ -355,6 +355,11 @@ internal void CheckMessage(Message message)
             {
                 throw ExceptionFactory.TooManyArgs(message.CommandAndKey, message.ArgCount);
             }
+
+            if (message.IsClientCaching && ClientSideTracking is null)
+            {
+                throw new InvalidOperationException("The " + nameof(CommandFlags.ClientCaching) + " flag can only be used if " + nameof(EnableServerAssistedClientSideTracking) + " has been called");
+            }
         }
 
         internal bool TryResend(int hashSlot, Message message, EndPoint endpoint, bool isMoved)
@@ -2268,7 +2273,7 @@ public async ValueTask DisposeAsync()
         public void Close(bool allowCommandsToComplete = true)
         {
             if (_isDisposed) return;
-
+            _clientSideTracking?.Shutdown();
             OnClosing(false);
             _isDisposed = true;
             _profilingSessionProvider = null;
@@ -2295,6 +2300,7 @@ public void Close(bool allowCommandsToComplete = true)
         public async Task CloseAsync(bool allowCommandsToComplete = true)
         {
             _isDisposed = true;
+            _clientSideTracking?.Shutdown();
             using (var tmp = pulse)
             {
                 pulse = null;

src/StackExchange.Redis/Enums/CommandFlags.cs

@@ -81,7 +81,10 @@ public enum CommandFlags
         /// </summary>
         NoScriptCache = 512,
 
-        // 1024: Removed - was used for async timeout checks; never user-specified, so not visible on the public API
+        /// <summary>
+        /// Indicates a command that relates to server-assisted client-side caching; this corresponds to CLIENT CACHING YES being issues before the command
+        /// </summary>
+        ClientCaching = 1024,
 
         // 2048: Use subscription connection type; never user-specified, so not visible on the public API
     }

src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs

@@ -281,5 +281,8 @@ public interface IConnectionMultiplexer : IDisposable, IAsyncDisposable
         /// <param name="destination">The destination stream to write the export to.</param>
         /// <param name="options">The options to use for this export.</param>
         void ExportConfiguration(Stream destination, ExportOptions options = ExportOptions.All);
+
+        /// <inheritdoc cref="ConnectionMultiplexer.EnableServerAssistedClientSideTracking(Func{RedisKey, ValueTask}, ClientTrackingOptions, ReadOnlyMemory{RedisKey})"/>
+        void EnableServerAssistedClientSideTracking(Func<RedisKey, ValueTask> keyInvalidated, ClientTrackingOptions options = ClientTrackingOptions.None, ReadOnlyMemory<RedisKey> prefixes = default);
     }
 }

src/StackExchange.Redis/Message.cs

@@ -75,7 +75,8 @@ internal abstract class Message : ICompletable
 #pragma warning restore CS0618
                                                        | CommandFlags.FireAndForget
                                                        | CommandFlags.NoRedirect
-                                                       | CommandFlags.NoScriptCache;
+                                                       | CommandFlags.NoScriptCache
+                                                       | CommandFlags.ClientCaching;
         private IResultBox? resultBox;
 
         private ResultProcessor? resultProcessor;
@@ -197,6 +198,7 @@ public bool IsAdmin
         }
 
         public bool IsAsking => (Flags & AskingFlag) != 0;
+        public bool IsClientCaching => (Flags & CommandFlags.ClientCaching) != 0;
 
         internal bool IsScriptUnavailable => (Flags & ScriptUnavailableFlag) != 0;
 

src/StackExchange.Redis/PhysicalBridge.cs

@@ -23,7 +23,9 @@ internal sealed class PhysicalBridge : IDisposable
 
         private const double ProfileLogSeconds = (1000 /* ms */ * ProfileLogSamples) / 1000.0;
 
-        private static readonly Message ReusableAskingCommand = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.ASKING);
+        private static readonly Message
+            ReusableAskingCommand = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.ASKING),
+            ReusableClientCachingYesCommand = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.CLIENT, RedisLiterals.CACHING, RedisLiterals.yes);
 
         private readonly long[] profileLog = new long[ProfileLogSamples];
 
@@ -1494,13 +1496,13 @@ private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection conne
                     }
                     if (message.IsAsking)
                     {
-                        var asking = ReusableAskingCommand;
-                        connection.EnqueueInsideWriteLock(asking);
-                        asking.WriteTo(connection);
-                        asking.SetRequestSent();
-                        IncrementOpCount();
+                        RawWriteInternalMessageInsideWriteLock(connection, ReusableAskingCommand);
                     }
                 }
+                if (message.IsClientCaching && connection.EnsureServerAssistedClientSideTrackingInsideWriteLock())
+                {
+                    RawWriteInternalMessageInsideWriteLock(connection, ReusableClientCachingYesCommand);
+                }
                 switch (cmd)
                 {
                     case RedisCommand.WATCH:
@@ -1570,6 +1572,15 @@ private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection conne
             }
         }
 
+        internal void RawWriteInternalMessageInsideWriteLock(PhysicalConnection connection, Message message)
+        {
+            message.SetInternalCall();
+            connection.EnqueueInsideWriteLock(message);
+            message.WriteTo(connection);
+            message.SetRequestSent();
+            IncrementOpCount();
+        }
+
         /// <summary>
         /// For testing only
         /// </summary>
@@ -1583,5 +1594,6 @@ internal void SimulateConnectionFailure(SimulatedFailureType failureType)
         }
 
         internal RedisCommand? GetActiveMessage() => Volatile.Read(ref _activeMessage)?.Command;
+        internal void OnSubscriberFailed() => physical?.OnSubscriberFailed();
     }
 }