From 2e84c37458b3bbf3c14246d413e2cace97d6ea4a Mon Sep 17 00:00:00 2001 From: jericho Date: Tue, 2 Jan 2024 11:30:27 -0500 Subject: [PATCH 01/28] Process the the remaining messages in the memory queue when shutting down Resolves #29 --- Source/Picton.Messaging/AsyncMessagePump.cs | 9 +++++---- Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 35dd09c..8859eb9 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -264,7 +264,9 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio // Define the task pump var pumpTask = Task.Run(async () => { - while (!cancellationToken.IsCancellationRequested) + // We process messages until cancellation is requested. + // When cancellation is requested, we continue processing messages until the memory queue is drained. + while (!cancellationToken.IsCancellationRequested || !queuedMessages.IsEmpty) { await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -274,8 +276,6 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio { var messageProcessed = false; - if (cancellationToken.IsCancellationRequested) return messageProcessed; - using (_metrics.Measure.Timer.Time(Metrics.MessageProcessingTimer)) { queuedMessages.TryDequeue(out CloudMessage message); @@ -331,7 +331,8 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio { semaphore.Release(); runningTasks.TryRemove(t, out Task taskToBeRemoved); - }, TaskContinuationOptions.ExecuteSynchronously) + }, + TaskContinuationOptions.ExecuteSynchronously) .IgnoreAwait(); } }); diff --git a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs index e4db98d..d67a7b9 100644 --- a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs @@ -296,7 +296,9 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio // Define the task pump var pumpTask = Task.Run(async () => { - while (!cancellationToken.IsCancellationRequested) + // We process messages until cancellation is requested. + // When cancellation is requested, we continue processing messages until the memory queue is drained. + while (!cancellationToken.IsCancellationRequested || !queuedMessages.IsEmpty) { await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -306,8 +308,6 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio { var messageProcessed = false; - if (cancellationToken.IsCancellationRequested) return messageProcessed; - using (_metrics.Measure.Timer.Time(Metrics.MessageProcessingTimer)) { if (queuedMessages.TryDequeue(out (string TenantId, CloudMessage Message) result)) @@ -363,7 +363,8 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio { semaphore.Release(); runningTasks.TryRemove(t, out Task taskToBeRemoved); - }, TaskContinuationOptions.ExecuteSynchronously) + }, + TaskContinuationOptions.ExecuteSynchronously) .IgnoreAwait(); } }); From 9042ad5b6050067de59d6f2ce400de08534242bb Mon Sep 17 00:00:00 2001 From: jericho Date: Tue, 2 Jan 2024 11:31:22 -0500 Subject: [PATCH 02/28] Distinct metrics for "empty tenant queue" vs. "ALL queues empty" Resolves #28 --- .../Picton.Messaging/AsyncMultiTenantMessagePump.cs | 2 +- Source/Picton.Messaging/Metrics.cs | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs index d67a7b9..8c7f3c9 100644 --- a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs @@ -450,7 +450,7 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio try { // All queues are empty - _metrics.Measure.Counter.Increment(Metrics.QueueEmptyCounter); + _metrics.Measure.Counter.Increment(Metrics.AllQueuesEmptyCounter); OnEmpty?.Invoke(cancellationToken); } catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) diff --git a/Source/Picton.Messaging/Metrics.cs b/Source/Picton.Messaging/Metrics.cs index 97c70d3..5c6dab6 100644 --- a/Source/Picton.Messaging/Metrics.cs +++ b/Source/Picton.Messaging/Metrics.cs @@ -36,7 +36,7 @@ internal static class Metrics }; /// - /// Gets the counter indicating the number of times we attempted to fetch messages from the Azure queue but the queue was empty. + /// Gets the counter indicating the number of times we attempted to fetch messages from an Azure queue but it was empty. /// public static CounterOptions QueueEmptyCounter => new CounterOptions { @@ -44,6 +44,15 @@ internal static class Metrics Name = "QueueEmptyCount" }; + /// + /// Gets the counter indicating the number of times we attempted to fetch messages from Azure but all the queues are empty. + /// + public static CounterOptions AllQueuesEmptyCounter => new CounterOptions + { + Context = "Picton", + Name = "AllQueuesEmptyCount" + }; + /// /// Gets the gauge indicating the number of messages waiting in the Azure queue over time. /// From 11d65038586f234e087638c0e6ed48982f57228d Mon Sep 17 00:00:00 2001 From: jericho Date: Wed, 3 Jan 2024 15:39:46 -0500 Subject: [PATCH 03/28] (doc) Add a comment in README.md to explain what is causing the AsyncMessagePump to stop --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 96f9f24..0db5d2e 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ namespace WorkerRole1 { Trace.TraceInformation("WorkerRole is stopping"); + // Invoking "Cancel()" will cause the AsyncMessagePump to stop this.cancellationTokenSource.Cancel(); this.runCompleteEvent.WaitOne(); From 4ed3703551a6da6720c29db7247e140d7ad9c7b6 Mon Sep 17 00:00:00 2001 From: jericho Date: Wed, 3 Jan 2024 15:43:00 -0500 Subject: [PATCH 04/28] (doc) formatting --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0db5d2e..15628de 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ namespace WorkerRole1 { Trace.TraceInformation("WorkerRole is stopping"); - // Invoking "Cancel()" will cause the AsyncMessagePump to stop + // Invoking "Cancel()" will cause the AsyncMessagePump to stop this.cancellationTokenSource.Cancel(); this.runCompleteEvent.WaitOne(); @@ -110,10 +110,10 @@ namespace WorkerRole1 var connectionString = "<-- insert connection string for your Azure account -->"; var queueName = "<-- insert the name of your Azure queue -->"; - // Configure the message pump + // Configure the message pump var messagePump = new AsyncMessagePump(connectionString, queueName, 10, null, TimeSpan.FromMinutes(1), 3) { - OnMessage = (message, cancellationToken) => + OnMessage = (message, cancellationToken) => { Debug.WriteLine("Received message of type {message.Content.GetType()}"); }, From d593413e58a7914328f1bce6115acc9858578ebd Mon Sep 17 00:00:00 2001 From: jericho Date: Wed, 3 Jan 2024 15:45:20 -0500 Subject: [PATCH 05/28] (doc) formatting --- README.md | 144 +++++++++++++++++++++++++++--------------------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 15628de..5e13d2a 100644 --- a/README.md +++ b/README.md @@ -55,78 +55,78 @@ using System.Diagnostics; namespace WorkerRole1 { - public class MyWorkerRole : RoleEntryPoint - { - private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - private readonly ManualResetEvent runCompleteEvent = new ManualResetEvent(false); - - public override void Run() - { - Trace.TraceInformation("WorkerRole is running"); - - try - { - this.RunAsync(this.cancellationTokenSource.Token).Wait(); - } - finally - { - this.runCompleteEvent.Set(); - } - } - - public override bool OnStart() - { - // Use TLS 1.2 for Service Bus connections - ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; - - // Set the maximum number of concurrent connections - ServicePointManager.DefaultConnectionLimit = 12; - - // For information on handling configuration changes - // see the MSDN topic at https://go.microsoft.com/fwlink/?LinkId=166357. - - bool result = base.OnStart(); - - Trace.TraceInformation("WorkerRole has been started"); - - return result; - } - - public override void OnStop() - { - Trace.TraceInformation("WorkerRole is stopping"); - - // Invoking "Cancel()" will cause the AsyncMessagePump to stop - this.cancellationTokenSource.Cancel(); - this.runCompleteEvent.WaitOne(); - - base.OnStop(); - - Trace.TraceInformation("WorkerRole has stopped"); - } - - private async Task RunAsync(CancellationToken cancellationToken) - { - var connectionString = "<-- insert connection string for your Azure account -->"; - var queueName = "<-- insert the name of your Azure queue -->"; - - // Configure the message pump - var messagePump = new AsyncMessagePump(connectionString, queueName, 10, null, TimeSpan.FromMinutes(1), 3) - { - OnMessage = (message, cancellationToken) => - { - Debug.WriteLine("Received message of type {message.Content.GetType()}"); - }, - OnError = (message, exception, isPoison) => - { - Trace.TraceInformation("An error occured: {0}", exception); - } - }; - - // Start the message pump - await messagePump.StartAsync(cancellationToken); - } - } + public class MyWorkerRole : RoleEntryPoint + { + private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + private readonly ManualResetEvent runCompleteEvent = new ManualResetEvent(false); + + public override void Run() + { + Trace.TraceInformation("WorkerRole is running"); + + try + { + this.RunAsync(this.cancellationTokenSource.Token).Wait(); + } + finally + { + this.runCompleteEvent.Set(); + } + } + + public override bool OnStart() + { + // Use TLS 1.2 for Service Bus connections + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + + // Set the maximum number of concurrent connections + ServicePointManager.DefaultConnectionLimit = 12; + + // For information on handling configuration changes + // see the MSDN topic at https://go.microsoft.com/fwlink/?LinkId=166357. + + bool result = base.OnStart(); + + Trace.TraceInformation("WorkerRole has been started"); + + return result; + } + + public override void OnStop() + { + Trace.TraceInformation("WorkerRole is stopping"); + + // Invoking "Cancel()" will cause the AsyncMessagePump to stop + this.cancellationTokenSource.Cancel(); + this.runCompleteEvent.WaitOne(); + + base.OnStop(); + + Trace.TraceInformation("WorkerRole has stopped"); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + var connectionString = "<-- insert connection string for your Azure account -->"; + var queueName = "<-- insert the name of your Azure queue -->"; + + // Configure the message pump + var messagePump = new AsyncMessagePump(connectionString, queueName, 10, null, TimeSpan.FromMinutes(1), 3) + { + OnMessage = (message, cancellationToken) => + { + Debug.WriteLine("Received message of type {message.Content.GetType()}"); + }, + OnError = (message, exception, isPoison) => + { + Trace.TraceInformation("An error occured: {0}", exception); + } + }; + + // Start the message pump + await messagePump.StartAsync(cancellationToken); + } + } } ``` From f8151160ad9e66d00edda1c4f6f6c19ec4511cf8 Mon Sep 17 00:00:00 2001 From: jericho Date: Thu, 4 Jan 2024 11:40:39 -0500 Subject: [PATCH 06/28] (doc) Better comments in README.md --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5e13d2a..f742f62 100644 --- a/README.md +++ b/README.md @@ -115,11 +115,14 @@ namespace WorkerRole1 { OnMessage = (message, cancellationToken) => { - Debug.WriteLine("Received message of type {message.Content.GetType()}"); + // This is where you insert your custom logic to process a message }, OnError = (message, exception, isPoison) => { - Trace.TraceInformation("An error occured: {0}", exception); + // Insert your custom error handling + + // Please note that the boolean "isPoison" parameter is an indication of + // whether this message will be automatically moved to the poison queue. } }; From a93b92530e5ff789c6f6ab7ecb0f6e48a615ff71 Mon Sep 17 00:00:00 2001 From: jericho Date: Thu, 4 Jan 2024 11:42:37 -0500 Subject: [PATCH 07/28] Formatting unit tests --- .../AsyncMessagePumpTests.cs | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs index c071d66..34597bb 100644 --- a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs +++ b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs @@ -241,18 +241,15 @@ public async Task Poison_message_is_rejected() .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => { - if (cloudMessage != null) + lock (lockObject) { - lock (lockObject) + if (cloudMessage != null) { - if (cloudMessage != null) - { - // DequeueCount is a private property. Therefore we must use reflection to change its value - var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); - dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message + // DequeueCount is a private property. Therefore we must use reflection to change its value + var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); + dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message - return Response.FromValue(new[] { cloudMessage }, new MockAzureResponse(200, "ok")); - } + return Response.FromValue(new[] { cloudMessage }, new MockAzureResponse(200, "ok")); } } return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); @@ -264,6 +261,7 @@ public async Task Poison_message_is_rejected() lock (lockObject) { cloudMessage = null; + isRejected = true; } return new MockAzureResponse(200, "ok"); }); @@ -280,14 +278,6 @@ public async Task Poison_message_is_rejected() OnError = (message, exception, isPoison) => { Interlocked.Increment(ref onErrorInvokeCount); - if (isPoison) - { - lock (lockObject) - { - isRejected = true; - cloudMessage = null; - } - } }, OnQueueEmpty = cancellationToken => { @@ -340,18 +330,15 @@ public async Task Poison_message_is_moved() .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => { - if (cloudMessage != null) + lock (lockObject) { - lock (lockObject) + if (cloudMessage != null) { - if (cloudMessage != null) - { - // DequeueCount is a private property. Therefore we must use reflection to change its value - var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); - dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message + // DequeueCount is a private property. Therefore we must use reflection to change its value + var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); + dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message - return Response.FromValue(new[] { cloudMessage }, new MockAzureResponse(200, "ok")); - } + return Response.FromValue(new[] { cloudMessage }, new MockAzureResponse(200, "ok")); } } return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); @@ -363,6 +350,7 @@ public async Task Poison_message_is_moved() lock (lockObject) { cloudMessage = null; + isRejected = true; } return new MockAzureResponse(200, "ok"); }); @@ -371,7 +359,6 @@ public async Task Poison_message_is_moved() .Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((string messageText, TimeSpan? visibilityTimeout, TimeSpan? timeToLive, CancellationToken cancellationToken) => { - // Nothing to do. We just want to ensure this method is invoked. var sendReceipt = QueuesModelFactory.SendReceipt("abc123", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddDays(7), "xyz", DateTimeOffset.UtcNow); return Response.FromValue(sendReceipt, new MockAzureResponse(200, "ok")); }); @@ -389,14 +376,6 @@ public async Task Poison_message_is_moved() OnError = (message, exception, isPoison) => { Interlocked.Increment(ref onErrorInvokeCount); - if (isPoison) - { - lock (lockObject) - { - isRejected = true; - cloudMessage = null; - } - } }, OnQueueEmpty = cancellationToken => { @@ -481,7 +460,7 @@ public async Task Exceptions_in_OnQueueEmpty_are_ignored() // Assert onMessageInvokeCount.ShouldBe(0); - onQueueEmptyInvokeCount.ShouldBeGreaterThan(0); + onQueueEmptyInvokeCount.ShouldBe(2); // First time we throw an exception, second time we stop the message pump onErrorInvokeCount.ShouldBe(0); mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); } From 894feb5f0e81e79f6b144639b53271edaaf08939 Mon Sep 17 00:00:00 2001 From: jericho Date: Thu, 4 Jan 2024 11:47:34 -0500 Subject: [PATCH 08/28] Fix how we determine if the max number of retries is reached Resolves #31 --- Source/Picton.Messaging/AsyncMessagePump.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 8859eb9..e8ac99a 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -293,7 +293,7 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio } catch (Exception ex) { - var isPoison = message.DequeueCount > _maxDequeueCount; + var isPoison = message.DequeueCount >= _maxDequeueCount; OnError?.Invoke(message, ex, isPoison); if (isPoison) { From 476ee42207a08fcef4ea9726d1eb5c053b7c26fc Mon Sep 17 00:00:00 2001 From: jericho Date: Thu, 4 Jan 2024 11:48:25 -0500 Subject: [PATCH 09/28] Ignore exceptions that occur in the custom error handler and add unit test to verify that such exceptions are indeed ignored Resolves #30 --- .../AsyncMessagePumpTests.cs | 84 +++++++++++++++++++ Source/Picton.Messaging/AsyncMessagePump.cs | 11 ++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs index 34597bb..82ae379 100644 --- a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs +++ b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs @@ -464,5 +464,89 @@ public async Task Exceptions_in_OnQueueEmpty_are_ignored() onErrorInvokeCount.ShouldBe(0); mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); } + + [Fact] + public async Task Exceptions_in_OnError_are_ignored() + { + // Arrange + var onMessageInvokeCount = 0; + var onQueueEmptyInvokeCount = 0; + var onErrorInvokeCount = 0; + + var lockObject = new Object(); + var cloudMessage = QueuesModelFactory.QueueMessage("abc123", "xyz", "Hello World!", 0, null, DateTimeOffset.UtcNow, null); + + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); + + var cts = new CancellationTokenSource(); + + mockQueueClient + .Setup(q => q.GetPropertiesAsync(It.IsAny())) + .ReturnsAsync((CancellationToken cancellationToken) => + { + var messageCount = cloudMessage == null ? 0 : 1; + var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); + return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => + { + lock (lockObject) + { + if (cloudMessage != null) + { + // DequeueCount is a private property. Therefore we must use reflection to change its value + var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); + dequeueCountProperty.SetValue(cloudMessage, cloudMessage.DequeueCount + 1); + + return Response.FromValue(new[] { cloudMessage }, new MockAzureResponse(200, "ok")); + } + } + return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string messageId, string popReceipt, CancellationToken cancellationToken) => + { + lock (lockObject) + { + cloudMessage = null; + } + return new MockAzureResponse(200, "ok"); + }); + + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + + var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) + { + OnMessage = (message, cancellationToken) => + { + Interlocked.Increment(ref onMessageInvokeCount); + throw new Exception("Simulate a problem while processing the message in order to unit test the error handler"); + }, + OnError = (message, exception, isPoison) => + { + Interlocked.Increment(ref onErrorInvokeCount); + throw new Exception("This dummy exception should be ignored"); + }, + OnQueueEmpty = cancellationToken => + { + Interlocked.Increment(ref onQueueEmptyInvokeCount); + cts.Cancel(); + } + }; + + // Act + await messagePump.StartAsync(cts.Token); + + // Assert + onMessageInvokeCount.ShouldBe(3); // <-- message is retried three times + onErrorInvokeCount.ShouldBe(3); // <-- we throw a dummy exception every time the mesage is processed, until we give up and the message is moved to the poison queue + onQueueEmptyInvokeCount.ShouldBe(1); + mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(4)); + mockQueueClient.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); + } } } diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index e8ac99a..63f2e28 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -294,7 +294,16 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio catch (Exception ex) { var isPoison = message.DequeueCount >= _maxDequeueCount; - OnError?.Invoke(message, ex, isPoison); + + try + { + OnError?.Invoke(message, ex, isPoison); + } + catch (Exception e) + { + _logger?.LogError(e.GetBaseException(), "An error occured when handling an exception. The error was caught and ignored."); + } + if (isPoison) { // PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue and moved to poison queue even when the message pump is shutting down From 0f48d82c784f39d9d318e7ccef19533b611fcd57 Mon Sep 17 00:00:00 2001 From: jericho Date: Thu, 4 Jan 2024 12:47:32 -0500 Subject: [PATCH 10/28] Replace Moq with NSubstitute Resolves #32 --- .../AsyncMessagePumpTests.cs | 116 ++++++++---------- .../Picton.Messaging.UnitTests/MockUtils.cs | 49 ++++---- .../Picton.Messaging.UnitTests.csproj | 6 +- 3 files changed, 82 insertions(+), 89 deletions(-) diff --git a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs index 82ae379..3de86a3 100644 --- a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs +++ b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs @@ -1,6 +1,6 @@ using Azure; using Azure.Storage.Queues.Models; -using Moq; +using NSubstitute; using Picton.Managers; using Shouldly; using System; @@ -29,7 +29,7 @@ public void Number_of_concurrent_tasks_too_small_throws() { var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object, false); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, false); var messagePump = new AsyncMessagePump(queueManager, null, 0, TimeSpan.FromMinutes(1), 1, null, null); }); @@ -42,7 +42,7 @@ public void DequeueCount_too_small_throws() { var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object, false); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, false); var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 0, null, null); }); @@ -54,7 +54,7 @@ public void Start_without_OnMessage_throws() // Arrange var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object, true); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, true); var cts = new CancellationTokenSource(); @@ -78,18 +78,17 @@ public async Task No_message_processed_when_queue_is_empty() var cts = new CancellationTokenSource(); mockQueueClient - .Setup(q => q.GetPropertiesAsync(It.IsAny())) - .ReturnsAsync((CancellationToken cancellationToken) => + .GetPropertiesAsync(Arg.Any()) + .Returns(callInfo => { var queueProperties = QueuesModelFactory.QueueProperties(null, 0); return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok"))) - .Verifiable(); + .ReceiveMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok"))); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) { @@ -115,7 +114,6 @@ public async Task No_message_processed_when_queue_is_empty() onMessageInvokeCount.ShouldBe(0); onQueueEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(0); - mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once()); } [Fact] @@ -135,17 +133,21 @@ public async Task Message_processed() var cts = new CancellationTokenSource(); mockQueueClient - .Setup(q => q.GetPropertiesAsync(It.IsAny())) - .ReturnsAsync((CancellationToken cancellationToken) => + .GetPropertiesAsync(Arg.Any()) + .Returns(callInfo => { var messageCount = cloudMessage == null ? 0 : 1; var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => + .ReceiveMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { + var maxMessages = callInfo.ArgAt(0); + var visibilityTimeout = callInfo.ArgAt(1); + var cancellationToken = callInfo.ArgAt(2); + if (cloudMessage != null) { lock (lockObject) @@ -163,8 +165,8 @@ public async Task Message_processed() return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((string messageId, string popReceipt, CancellationToken cancellationToken) => + .DeleteMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { lock (lockObject) { @@ -173,7 +175,7 @@ public async Task Message_processed() return new MockAzureResponse(200, "ok"); }); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) { @@ -206,8 +208,6 @@ public async Task Message_processed() onMessageInvokeCount.ShouldBe(1); onQueueEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(0); - mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); - mockQueueClient.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Fact] @@ -230,16 +230,16 @@ public async Task Poison_message_is_rejected() var cts = new CancellationTokenSource(); mockQueueClient - .Setup(q => q.GetPropertiesAsync(It.IsAny())) - .ReturnsAsync((CancellationToken cancellationToken) => - { - var messageCount = cloudMessage == null ? 0 : 1; - var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); - return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); - }); + .GetPropertiesAsync(Arg.Any()) + .Returns(callInfo => + { + var messageCount = cloudMessage == null ? 0 : 1; + var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); + return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); + }); mockQueueClient - .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => + .ReceiveMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { lock (lockObject) { @@ -255,8 +255,8 @@ public async Task Poison_message_is_rejected() return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((string messageId, string popReceipt, CancellationToken cancellationToken) => + .DeleteMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { lock (lockObject) { @@ -266,7 +266,7 @@ public async Task Poison_message_is_rejected() return new MockAzureResponse(200, "ok"); }); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) { @@ -294,8 +294,6 @@ public async Task Poison_message_is_rejected() onQueueEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(1); isRejected.ShouldBeTrue(); - mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); - mockQueueClient.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Fact] @@ -319,16 +317,16 @@ public async Task Poison_message_is_moved() var cts = new CancellationTokenSource(); mockQueueClient - .Setup(q => q.GetPropertiesAsync(It.IsAny())) - .ReturnsAsync((CancellationToken cancellationToken) => + .GetPropertiesAsync(Arg.Any()) + .Returns(callInfo => { var messageCount = cloudMessage == null ? 0 : 1; var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => + .ReceiveMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { lock (lockObject) { @@ -344,8 +342,8 @@ public async Task Poison_message_is_moved() return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((string messageId, string popReceipt, CancellationToken cancellationToken) => + .DeleteMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { lock (lockObject) { @@ -356,15 +354,15 @@ public async Task Poison_message_is_moved() }); mockPoisonQueueClient - .Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((string messageText, TimeSpan? visibilityTimeout, TimeSpan? timeToLive, CancellationToken cancellationToken) => + .SendMessageAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { var sendReceipt = QueuesModelFactory.SendReceipt("abc123", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddDays(7), "xyz", DateTimeOffset.UtcNow); return Response.FromValue(sendReceipt, new MockAzureResponse(200, "ok")); }); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); - var poisonQueueManager = new QueueManager(mockBlobContainerClient.Object, mockPoisonQueueClient.Object); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); + var poisonQueueManager = new QueueManager(mockBlobContainerClient, mockPoisonQueueClient); var messagePump = new AsyncMessagePump(queueManager, poisonQueueManager, 1, TimeSpan.FromMinutes(1), 3, null) { @@ -392,9 +390,6 @@ public async Task Poison_message_is_moved() onQueueEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(1); isRejected.ShouldBeTrue(); - mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); - mockQueueClient.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); - mockPoisonQueueClient.Verify(q => q.SendMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Fact] @@ -414,17 +409,17 @@ public async Task Exceptions_in_OnQueueEmpty_are_ignored() var cts = new CancellationTokenSource(); mockQueueClient - .Setup(q => q.GetPropertiesAsync(It.IsAny())) - .ReturnsAsync((CancellationToken cancellationToken) => + .GetPropertiesAsync(Arg.Any()) + .Returns(callInfo => { var queueProperties = QueuesModelFactory.QueueProperties(null, 0); return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok"))); + .ReceiveMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok"))); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null, null) { @@ -462,7 +457,6 @@ public async Task Exceptions_in_OnQueueEmpty_are_ignored() onMessageInvokeCount.ShouldBe(0); onQueueEmptyInvokeCount.ShouldBe(2); // First time we throw an exception, second time we stop the message pump onErrorInvokeCount.ShouldBe(0); - mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); } [Fact] @@ -482,16 +476,16 @@ public async Task Exceptions_in_OnError_are_ignored() var cts = new CancellationTokenSource(); mockQueueClient - .Setup(q => q.GetPropertiesAsync(It.IsAny())) - .ReturnsAsync((CancellationToken cancellationToken) => + .GetPropertiesAsync(Arg.Any()) + .Returns(callInfo => { var messageCount = cloudMessage == null ? 0 : 1; var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => + .ReceiveMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { lock (lockObject) { @@ -507,8 +501,8 @@ public async Task Exceptions_in_OnError_are_ignored() return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); }); mockQueueClient - .Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((string messageId, string popReceipt, CancellationToken cancellationToken) => + .DeleteMessageAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => { lock (lockObject) { @@ -517,7 +511,7 @@ public async Task Exceptions_in_OnError_are_ignored() return new MockAzureResponse(200, "ok"); }); - var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) { @@ -545,8 +539,6 @@ public async Task Exceptions_in_OnError_are_ignored() onMessageInvokeCount.ShouldBe(3); // <-- message is retried three times onErrorInvokeCount.ShouldBe(3); // <-- we throw a dummy exception every time the mesage is processed, until we give up and the message is moved to the poison queue onQueueEmptyInvokeCount.ShouldBe(1); - mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(4)); - mockQueueClient.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); } } } diff --git a/Source/Picton.Messaging.UnitTests/MockUtils.cs b/Source/Picton.Messaging.UnitTests/MockUtils.cs index bcf3ca8..6d1f1c7 100644 --- a/Source/Picton.Messaging.UnitTests/MockUtils.cs +++ b/Source/Picton.Messaging.UnitTests/MockUtils.cs @@ -2,7 +2,7 @@ using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Queues; -using Moq; +using NSubstitute; using System; using System.Collections.Generic; using System.Linq; @@ -15,73 +15,70 @@ internal static class MockUtils private static readonly string QUEUE_STORAGE_URL = "http://bogus:10001/devstoreaccount1/"; private static readonly string BLOB_STORAGE_URL = "http://bogus:10002/devstoreaccount1/"; - internal static Mock GetMockBlobContainerClient(string containerName = "mycontainer", IEnumerable> mockBlobClients = null) + internal static BlobContainerClient GetMockBlobContainerClient(string containerName = "mycontainer", IEnumerable mockBlobClients = null) { var mockContainerUri = new Uri(BLOB_STORAGE_URL + containerName); var blobContainerInfo = BlobsModelFactory.BlobContainerInfo(ETag.All, DateTimeOffset.UtcNow); - var mockBlobContainer = new Mock(MockBehavior.Strict); + var mockBlobContainer = Substitute.For(); mockBlobContainer - .SetupGet(m => m.Name) + .Name .Returns(containerName); mockBlobContainer - .SetupGet(m => m.Uri) + .Uri .Returns(mockContainerUri); mockBlobContainer - .Setup(c => c.CreateIfNotExists(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) - .Returns(Response.FromValue(blobContainerInfo, new MockAzureResponse(200, "ok"))) - .Verifiable(); + .CreateIfNotExists(Arg.Any(), Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(Response.FromValue(blobContainerInfo, new MockAzureResponse(200, "ok"))); - foreach (var blobClient in mockBlobClients?.Select(m => m.Object) ?? Enumerable.Empty()) + foreach (var blobClient in mockBlobClients ?? Enumerable.Empty()) { mockBlobContainer - .Setup(c => c.GetBlobClient(blobClient.Name)) - .Returns(blobClient) - .Verifiable(); + .GetBlobClient(blobClient.Name) + .Returns(blobClient); } return mockBlobContainer; } - internal static Mock GetMockBlobClient(string blobName) + internal static BlobClient GetMockBlobClient(string blobName) { var mockBlobUri = new Uri(BLOB_STORAGE_URL + blobName); - var mockBlobClient = new Mock(MockBehavior.Strict); + var mockBlobClient = Substitute.For(); mockBlobClient - .SetupGet(m => m.Name) + .Name .Returns(blobName); mockBlobClient - .SetupGet(m => m.Uri) + .Uri .Returns(mockBlobUri); return mockBlobClient; } - internal static Mock GetMockQueueClient(string queueName = "myqueue") + internal static QueueClient GetMockQueueClient(string queueName = "myqueue") { var mockQueueStorageUri = new Uri(QUEUE_STORAGE_URL + queueName); - var mockQueueClient = new Mock(MockBehavior.Strict); + var mockQueueClient = Substitute.For(); mockQueueClient - .SetupGet(m => m.MaxPeekableMessages) - .Returns(32); + .Uri + .Returns(mockQueueStorageUri); mockQueueClient - .SetupGet(m => m.MessageMaxBytes) + .MessageMaxBytes .Returns(65536); mockQueueClient - .SetupGet(m => m.Uri) - .Returns(mockQueueStorageUri); + .MaxPeekableMessages + .Returns(32); mockQueueClient - .Setup(c => c.CreateIfNotExists(It.IsAny>(), It.IsAny())) - .Returns((Response)null) - .Verifiable(); + .CreateIfNotExists(Arg.Any>(), Arg.Any()) + .Returns((Response)null); return mockQueueClient; } diff --git a/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj b/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj index 29bff2a..7936cc2 100644 --- a/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj +++ b/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj @@ -12,7 +12,11 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + From 4feeff4a48f336a4483b69031fa710c43f82d417 Mon Sep 17 00:00:00 2001 From: jericho Date: Fri, 5 Jan 2024 11:21:02 -0500 Subject: [PATCH 11/28] (doc) better comment in README.md --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f742f62..9d04d24 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,21 @@ namespace WorkerRole1 { // Insert your custom error handling - // Please note that the boolean "isPoison" parameter is an indication of - // whether this message will be automatically moved to the poison queue. + // ========================================================================== + // Important note regarding "isPoison": + // -------------------------------------------------------------------------- + // this parameter indicates whether this message has exceeded the maximum + // number of retries. + // + // When you have configured the "poison queue name" and this parameter is + // "true", the message is automatically copied to the poison queue and + // removed from the original queue. + // + // If you have not configured the "poison queue name" and this parameter is + // "true", the message is automatically removed from the original queue and + // you are responsible for storing the message. If you don't, this mesage + // will be lost. + // ========================================================================== } }; From ac88c85469f5ba377b93d3a9827f4efe56e752ef Mon Sep 17 00:00:00 2001 From: jericho Date: Fri, 5 Jan 2024 11:21:49 -0500 Subject: [PATCH 12/28] Minor improvements --- Source/Picton.Messaging/Extensions.cs | 2 +- .../Properties/AssemblyInfo.cs | 3 +- .../Utilities/MessageHandlersDiscoverer.cs | 87 +++++++++++++++++++ .../Utilities/RoundRobinList.cs | 12 ++- 4 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 Source/Picton.Messaging/Utilities/MessageHandlersDiscoverer.cs diff --git a/Source/Picton.Messaging/Extensions.cs b/Source/Picton.Messaging/Extensions.cs index 6b7c5cf..cba4d8b 100644 --- a/Source/Picton.Messaging/Extensions.cs +++ b/Source/Picton.Messaging/Extensions.cs @@ -7,7 +7,7 @@ namespace Picton.Messaging /// /// Extension methods. /// - public static class Extensions + internal static class Extensions { #region PUBLIC EXTENSION METHODS diff --git a/Source/Picton.Messaging/Properties/AssemblyInfo.cs b/Source/Picton.Messaging/Properties/AssemblyInfo.cs index de29e15..b874b1c 100644 --- a/Source/Picton.Messaging/Properties/AssemblyInfo.cs +++ b/Source/Picton.Messaging/Properties/AssemblyInfo.cs @@ -1,7 +1,8 @@ -using System.Runtime.CompilerServices; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; [assembly: InternalsVisibleTo("Picton.Messaging.UnitTests")] +[assembly: InternalsVisibleTo("Picton.Messaging.IntegrationTests")] // Setting ComVisible to false makes the types in this assembly not visible // to COM components. If you need to access a type in this assembly from diff --git a/Source/Picton.Messaging/Utilities/MessageHandlersDiscoverer.cs b/Source/Picton.Messaging/Utilities/MessageHandlersDiscoverer.cs new file mode 100644 index 0000000..c82664d --- /dev/null +++ b/Source/Picton.Messaging/Utilities/MessageHandlersDiscoverer.cs @@ -0,0 +1,87 @@ +using Microsoft.Extensions.DependencyModel; +using Microsoft.Extensions.Logging; +using Picton.Messaging.Messages; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace Picton.Messaging.Utilities +{ + internal static class MessageHandlersDiscoverer + { + public static IDictionary GetMessageHandlers(ILogger logger) + { + logger?.LogTrace("Discovering message handlers."); + + var assemblies = GetLocalAssemblies(); + + var assembliesCount = assemblies.Length; + if (assembliesCount == 0) logger?.LogTrace($"Did not find any local assembly."); + else if (assembliesCount == 1) logger?.LogTrace("Found 1 local assembly."); + else logger?.LogTrace($"Found {assemblies.Count()} local assemblies."); + + var typesWithMessageHandlerInterfaces = assemblies + .SelectMany(x => x.GetTypes()) + .Where(t => !t.GetTypeInfo().IsInterface) + .Select(type => new + { + Type = type, + MessageTypes = type + .GetInterfaces() + .Where(i => i.GetTypeInfo().IsGenericType) + .Where(i => i.GetGenericTypeDefinition() == typeof(IMessageHandler<>)) + .SelectMany(i => i.GetGenericArguments()) + }) + .Where(t => t.MessageTypes != null && t.MessageTypes.Any()) + .ToArray(); + + var classesCount = typesWithMessageHandlerInterfaces.Length; + if (classesCount == 0) logger?.LogTrace("Did not find any class implementing the 'IMessageHandler' interface."); + else if (classesCount == 1) logger?.LogTrace("Found 1 class implementing the 'IMessageHandler' interface."); + else logger?.LogTrace($"Found {typesWithMessageHandlerInterfaces.Count()} classes implementing the 'IMessageHandler' interface."); + + var oneTypePerMessageHandler = typesWithMessageHandlerInterfaces + .SelectMany(t => t.MessageTypes, (t, messageType) => + new + { + t.Type, + MessageType = messageType + }) + .ToArray(); + + var messageHandlers = oneTypePerMessageHandler + .GroupBy(h => h.MessageType) + .ToDictionary(group => group.Key, group => group.Select(t => t.Type) + .ToArray()); + + return messageHandlers; + } + + private static Assembly[] GetLocalAssemblies() + { + var dependencies = DependencyContext.Default.RuntimeLibraries; + + var assemblies = new List(); + foreach (var library in dependencies) + { + if (IsCandidateLibrary(library)) + { + var assembly = Assembly.Load(new AssemblyName(library.Name)); + assemblies.Add(assembly); + } + } + + return assemblies.ToArray(); + } + + private static bool IsCandidateLibrary(RuntimeLibrary library) + { + return !library.Name.StartsWith("Microsoft.", StringComparison.OrdinalIgnoreCase) && + !library.Name.StartsWith("System.", StringComparison.OrdinalIgnoreCase) && + !library.Name.StartsWith("NetStandard.", StringComparison.OrdinalIgnoreCase) && + !string.Equals(library.Type, "package", StringComparison.OrdinalIgnoreCase) && + !string.Equals(library.Type, "referenceassembly", StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/Source/Picton.Messaging/Utilities/RoundRobinList.cs b/Source/Picton.Messaging/Utilities/RoundRobinList.cs index 702f47d..f13d588 100644 --- a/Source/Picton.Messaging/Utilities/RoundRobinList.cs +++ b/Source/Picton.Messaging/Utilities/RoundRobinList.cs @@ -20,13 +20,11 @@ public RoundRobinList(IEnumerable list) _linkedList = new LinkedList(list); } - public T Current - { - get - { - return _current == default ? default : _current.Value; - } - } + public T Current => _current == default ? default : _current.Value; + + public T Next => _current == default ? default : _current.Next.Value; + + public T Previous => _current == default ? default : _current.Previous.Value; public int Count => _linkedList.Count; From 61bef01cf25fbed2aa9ef77a71027bd621865fd0 Mon Sep 17 00:00:00 2001 From: jericho Date: Fri, 5 Jan 2024 11:30:37 -0500 Subject: [PATCH 13/28] AsyncMessagePump should handle multiple queues --- .../TestsRunner.cs | 12 +- .../AsyncMessagePumpTests.cs | 115 ++--- Source/Picton.Messaging/AsyncMessagePump.cs | 429 +++++++++++++----- .../AsyncMessagePumpWithHandlers.cs | 237 ++++------ Source/Picton.Messaging/MessagePumpOptions.cs | 42 ++ Source/Picton.Messaging/QueueConfig.cs | 23 + 6 files changed, 516 insertions(+), 342 deletions(-) create mode 100644 Source/Picton.Messaging/MessagePumpOptions.cs create mode 100644 Source/Picton.Messaging/QueueConfig.cs diff --git a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs index f2172c1..10afb66 100644 --- a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs +++ b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs @@ -113,9 +113,10 @@ private async Task RunAsyncMessagePumpTests(string connectionString, string queu Stopwatch sw = null; // Configure the message pump - var messagePump = new AsyncMessagePump(connectionString, queueName, 10, null, TimeSpan.FromMinutes(1), 3, _logger, metrics) + var options = new MessagePumpOptions(connectionString, 10, null, null); + var messagePump = new AsyncMessagePump(options, queueName, null, TimeSpan.FromMinutes(1), 3, _logger, metrics) { - OnMessage = (message, cancellationToken) => + OnMessage = (queueName, message, cancellationToken) => { _logger.LogInformation(message.Content.ToString()); } @@ -123,7 +124,7 @@ private async Task RunAsyncMessagePumpTests(string connectionString, string queu // Stop the message pump when the queue is empty. var cts = new CancellationTokenSource(); - messagePump.OnQueueEmpty = cancellationToken => + messagePump.OnEmpty = cancellationToken => { // Stop the timer if (sw.IsRunning) sw.Stop(); @@ -163,8 +164,9 @@ private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, // Configure the message pump var cts = new CancellationTokenSource(); - var messagePump = new AsyncMessagePumpWithHandlers(connectionString, queueName, 10, null, TimeSpan.FromMinutes(1), 3, _logger, metrics); - messagePump.OnQueueEmpty = cancellationToken => + var options = new MessagePumpOptions(connectionString, 10, null, null); + var messagePump = new AsyncMessagePumpWithHandlers(options, queueName, null, TimeSpan.FromMinutes(1), 3, _logger, metrics); + messagePump.OnEmpty = cancellationToken => { // Stop the timer if (sw.IsRunning) sw.Stop(); diff --git a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs index 3de86a3..6e63d7f 100644 --- a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs +++ b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs @@ -16,9 +16,11 @@ public class AsyncMessagePumpTests [Fact] public void Null_cloudQueue_throws() { + var options = new MessagePumpOptions("bogus connection string", 1); + Should.Throw(() => { - var messagePump = new AsyncMessagePump((QueueManager)null, null, 1, TimeSpan.FromMinutes(1), 1, null, null); + var messagePump = new AsyncMessagePump(options, (QueueManager)null); }); } @@ -30,8 +32,9 @@ public void Number_of_concurrent_tasks_too_small_throws() var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, false); + var options = new MessagePumpOptions("bogus connection string", 0); - var messagePump = new AsyncMessagePump(queueManager, null, 0, TimeSpan.FromMinutes(1), 1, null, null); + var messagePump = new AsyncMessagePump(options, queueManager); }); } @@ -43,8 +46,9 @@ public void DequeueCount_too_small_throws() var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, false); + var options = new MessagePumpOptions("bogus connection string", 0); - var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 0, null, null); + var messagePump = new AsyncMessagePump(options, queueManager, maxDequeueCount: 0); }); } @@ -55,10 +59,11 @@ public void Start_without_OnMessage_throws() var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, true); + var options = new MessagePumpOptions("bogus connection string", 1); var cts = new CancellationTokenSource(); - var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null, null); + var messagePump = new AsyncMessagePump(options, queueManager); // Act Should.ThrowAsync(() => messagePump.StartAsync(cts.Token)); @@ -69,7 +74,7 @@ public async Task No_message_processed_when_queue_is_empty() { // Arrange var onMessageInvokeCount = 0; - var onQueueEmptyInvokeCount = 0; + var OnEmptyInvokeCount = 0; var onErrorInvokeCount = 0; var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); @@ -89,20 +94,21 @@ public async Task No_message_processed_when_queue_is_empty() .Returns(callInfo => Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok"))); var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); + var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) + var messagePump = new AsyncMessagePump(options, queueManager) { - OnMessage = (message, cancellationToken) => + OnMessage = (queueName, message, cancellationToken) => { Interlocked.Increment(ref onMessageInvokeCount); }, - OnError = (message, exception, isPoison) => + OnError = (queueName, message, exception, isPoison) => { Interlocked.Increment(ref onErrorInvokeCount); }, - OnQueueEmpty = cancellationToken => + OnEmpty = cancellationToken => { - Interlocked.Increment(ref onQueueEmptyInvokeCount); + Interlocked.Increment(ref OnEmptyInvokeCount); cts.Cancel(); } }; @@ -112,7 +118,7 @@ public async Task No_message_processed_when_queue_is_empty() // Assert onMessageInvokeCount.ShouldBe(0); - onQueueEmptyInvokeCount.ShouldBe(1); + OnEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(0); } @@ -121,7 +127,7 @@ public async Task Message_processed() { // Arrange var onMessageInvokeCount = 0; - var onQueueEmptyInvokeCount = 0; + var OnEmptyInvokeCount = 0; var onErrorInvokeCount = 0; var lockObject = new Object(); @@ -154,7 +160,7 @@ public async Task Message_processed() { if (cloudMessage != null) { - // DequeueCount is a private property. Therefore we must use reflection to change its value + // DequeueCount is a read-only property but we can use reflection to change its value var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); dequeueCountProperty.SetValue(cloudMessage, cloudMessage.DequeueCount + 1); @@ -176,14 +182,15 @@ public async Task Message_processed() }); var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); + var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) + var messagePump = new AsyncMessagePump(options, queueManager) { - OnMessage = (message, cancellationToken) => + OnMessage = (queueName, message, cancellationToken) => { Interlocked.Increment(ref onMessageInvokeCount); }, - OnError = (message, exception, isPoison) => + OnError = (queueName, message, exception, isPoison) => { Interlocked.Increment(ref onErrorInvokeCount); if (isPoison) @@ -194,9 +201,9 @@ public async Task Message_processed() } } }, - OnQueueEmpty = cancellationToken => + OnEmpty = cancellationToken => { - Interlocked.Increment(ref onQueueEmptyInvokeCount); + Interlocked.Increment(ref OnEmptyInvokeCount); cts.Cancel(); } }; @@ -206,7 +213,7 @@ public async Task Message_processed() // Assert onMessageInvokeCount.ShouldBe(1); - onQueueEmptyInvokeCount.ShouldBe(1); + OnEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(0); } @@ -215,7 +222,7 @@ public async Task Poison_message_is_rejected() { // Arrange var onMessageInvokeCount = 0; - var onQueueEmptyInvokeCount = 0; + var OnEmptyInvokeCount = 0; var onErrorInvokeCount = 0; var isRejected = false; @@ -245,7 +252,7 @@ public async Task Poison_message_is_rejected() { if (cloudMessage != null) { - // DequeueCount is a private property. Therefore we must use reflection to change its value + // DequeueCount is a read-only property but we can use reflection to change its value var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message @@ -267,21 +274,22 @@ public async Task Poison_message_is_rejected() }); var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); + var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) + var messagePump = new AsyncMessagePump(options, queueManager) { - OnMessage = (message, cancellationToken) => + OnMessage = (queueName, message, cancellationToken) => { Interlocked.Increment(ref onMessageInvokeCount); throw new Exception("An error occured when attempting to process the message"); }, - OnError = (message, exception, isPoison) => + OnError = (queueName, message, exception, isPoison) => { Interlocked.Increment(ref onErrorInvokeCount); }, - OnQueueEmpty = cancellationToken => + OnEmpty = cancellationToken => { - Interlocked.Increment(ref onQueueEmptyInvokeCount); + Interlocked.Increment(ref OnEmptyInvokeCount); cts.Cancel(); } }; @@ -291,7 +299,7 @@ public async Task Poison_message_is_rejected() // Assert onMessageInvokeCount.ShouldBe(1); - onQueueEmptyInvokeCount.ShouldBe(1); + OnEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(1); isRejected.ShouldBeTrue(); } @@ -301,7 +309,7 @@ public async Task Poison_message_is_moved() { // Arrange var onMessageInvokeCount = 0; - var onQueueEmptyInvokeCount = 0; + var OnEmptyInvokeCount = 0; var onErrorInvokeCount = 0; var isRejected = false; @@ -332,7 +340,7 @@ public async Task Poison_message_is_moved() { if (cloudMessage != null) { - // DequeueCount is a private property. Therefore we must use reflection to change its value + // DequeueCount is a read-only property but we can use reflection to change its value var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message @@ -363,21 +371,22 @@ public async Task Poison_message_is_moved() var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var poisonQueueManager = new QueueManager(mockBlobContainerClient, mockPoisonQueueClient); + var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(queueManager, poisonQueueManager, 1, TimeSpan.FromMinutes(1), 3, null) + var messagePump = new AsyncMessagePump(options, queueManager, poisonQueueManager) { - OnMessage = (message, cancellationToken) => + OnMessage = (queueName, message, cancellationToken) => { Interlocked.Increment(ref onMessageInvokeCount); throw new Exception("An error occured when attempting to process the message"); }, - OnError = (message, exception, isPoison) => + OnError = (queueName, message, exception, isPoison) => { Interlocked.Increment(ref onErrorInvokeCount); }, - OnQueueEmpty = cancellationToken => + OnEmpty = cancellationToken => { - Interlocked.Increment(ref onQueueEmptyInvokeCount); + Interlocked.Increment(ref OnEmptyInvokeCount); cts.Cancel(); } }; @@ -387,17 +396,17 @@ public async Task Poison_message_is_moved() // Assert onMessageInvokeCount.ShouldBe(1); - onQueueEmptyInvokeCount.ShouldBe(1); + OnEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(1); isRejected.ShouldBeTrue(); } [Fact] - public async Task Exceptions_in_OnQueueEmpty_are_ignored() + public async Task Exceptions_in_OnEmpty_are_ignored() { // Arrange var onMessageInvokeCount = 0; - var onQueueEmptyInvokeCount = 0; + var OnEmptyInvokeCount = 0; var onErrorInvokeCount = 0; var exceptionSimulated = false; @@ -420,20 +429,21 @@ public async Task Exceptions_in_OnQueueEmpty_are_ignored() .Returns(callInfo => Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok"))); var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); + var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null, null) + var messagePump = new AsyncMessagePump(options, queueManager) { - OnMessage = (message, cancellationToken) => + OnMessage = (queueName, message, cancellationToken) => { Interlocked.Increment(ref onMessageInvokeCount); }, - OnError = (message, exception, isPoison) => + OnError = (queueName, message, exception, isPoison) => { Interlocked.Increment(ref onErrorInvokeCount); }, - OnQueueEmpty = cancellationToken => + OnEmpty = cancellationToken => { - Interlocked.Increment(ref onQueueEmptyInvokeCount); + Interlocked.Increment(ref OnEmptyInvokeCount); // Simulate an exception (only the first time) lock (lockObject) @@ -455,7 +465,7 @@ public async Task Exceptions_in_OnQueueEmpty_are_ignored() // Assert onMessageInvokeCount.ShouldBe(0); - onQueueEmptyInvokeCount.ShouldBe(2); // First time we throw an exception, second time we stop the message pump + OnEmptyInvokeCount.ShouldBe(2); // First time we throw an exception, second time we stop the message pump onErrorInvokeCount.ShouldBe(0); } @@ -464,7 +474,7 @@ public async Task Exceptions_in_OnError_are_ignored() { // Arrange var onMessageInvokeCount = 0; - var onQueueEmptyInvokeCount = 0; + var OnEmptyInvokeCount = 0; var onErrorInvokeCount = 0; var lockObject = new Object(); @@ -491,7 +501,7 @@ public async Task Exceptions_in_OnError_are_ignored() { if (cloudMessage != null) { - // DequeueCount is a private property. Therefore we must use reflection to change its value + // DequeueCount is a read-only property but we can use reflection to change its value var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); dequeueCountProperty.SetValue(cloudMessage, cloudMessage.DequeueCount + 1); @@ -512,22 +522,23 @@ public async Task Exceptions_in_OnError_are_ignored() }); var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); + var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) + var messagePump = new AsyncMessagePump(options, queueManager) { - OnMessage = (message, cancellationToken) => + OnMessage = (queueName, message, cancellationToken) => { Interlocked.Increment(ref onMessageInvokeCount); throw new Exception("Simulate a problem while processing the message in order to unit test the error handler"); }, - OnError = (message, exception, isPoison) => + OnError = (queueName, message, exception, isPoison) => { Interlocked.Increment(ref onErrorInvokeCount); throw new Exception("This dummy exception should be ignored"); }, - OnQueueEmpty = cancellationToken => + OnEmpty = cancellationToken => { - Interlocked.Increment(ref onQueueEmptyInvokeCount); + Interlocked.Increment(ref OnEmptyInvokeCount); cts.Cancel(); } }; @@ -537,8 +548,8 @@ public async Task Exceptions_in_OnError_are_ignored() // Assert onMessageInvokeCount.ShouldBe(3); // <-- message is retried three times - onErrorInvokeCount.ShouldBe(3); // <-- we throw a dummy exception every time the mesage is processed, until we give up and the message is moved to the poison queue - onQueueEmptyInvokeCount.ShouldBe(1); + onErrorInvokeCount.ShouldBe(3); // <-- we throw a dummy exception every time the mesage is processed, until we give up and move the message to the poison queue + OnEmptyInvokeCount.ShouldBe(1); } } } diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 63f2e28..bfda366 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -1,12 +1,13 @@ using App.Metrics; +using Azure; using Microsoft.Extensions.Logging; -using Picton.Interfaces; using Picton.Managers; using Picton.Messaging.Utilities; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -14,17 +15,17 @@ namespace Picton.Messaging { /// /// High performance message processor (also known as a message "pump") for Azure storage queues. - /// Designed to monitor an Azure storage queue and process the message as quickly and efficiently as possible. + /// Designed to monitor either a single queue or a fixed list of queues and process messages as + /// quickly and efficiently as possible. /// public class AsyncMessagePump { #region FIELDS - private readonly IQueueManager _queueManager; - private readonly IQueueManager _poisonQueueManager; - private readonly int _concurrentTasks; - private readonly TimeSpan? _visibilityTimeout; - private readonly int _maxDequeueCount; + private readonly ConcurrentDictionary _queueManagers = new ConcurrentDictionary(); + private readonly RoundRobinList _queueNames; + + private readonly MessagePumpOptions _mesagePumpOptions; private readonly ILogger _logger; private readonly IMetrics _metrics; @@ -38,7 +39,7 @@ public class AsyncMessagePump /// /// If exception is thrown when calling OnMessage, it will regard this queue message as failed. /// - public Action OnMessage { get; set; } + public Action OnMessage { get; set; } /// /// Gets or sets the logic to execute when an error occurs. @@ -51,20 +52,20 @@ public class AsyncMessagePump /// /// When isPoison is set to true, you should copy this message to a poison queue because it will be deleted from the original queue. /// - public Action OnError { get; set; } + public Action OnError { get; set; } /// - /// Gets or sets the logic to execute when queue is empty. + /// Gets or sets the logic to execute when all queues are empty. /// /// /// - /// OnQueueEmpty = cancellationToken => Task.Delay(2500, cancellationToken).Wait(); + /// OnEmpty = cancellationToken => Task.Delay(2500, cancellationToken).Wait(); /// /// /// - /// If this property is not set, the default logic is to pause for 1.5 seconds. + /// If this property is not set, the default logic is to do nothing. /// - public Action OnQueueEmpty { get; set; } + public Action OnEmpty { get; set; } #endregion @@ -78,42 +79,101 @@ public class AsyncMessagePump /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. /// /// Name of the queue. - /// The number of concurrent tasks. + /// /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. /// The visibility timeout. /// The maximum dequeue count. /// The logger. /// The system where metrics are published. [ExcludeFromCodeCoverage] - public AsyncMessagePump(string connectionString, string queueName, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) - : this(new QueueManager(connectionString, queueName), string.IsNullOrEmpty(poisonQueueName) ? null : new QueueManager(connectionString, poisonQueueName), concurrentTasks, visibilityTimeout, maxDequeueCount, logger, metrics) + public AsyncMessagePump(MessagePumpOptions options, string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + : this(options, new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount), logger, metrics) { } /// /// Initializes a new instance of the class. /// + /// + /// + /// The logger. + /// The system where metrics are published. + [ExcludeFromCodeCoverage] + public AsyncMessagePump(MessagePumpOptions options, QueueConfig queueConfig, ILogger logger = null, IMetrics metrics = null) + : this(options, new[] { queueConfig }, logger, metrics) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The configuration options for each queue to be monitored. + /// The logger. + /// The system where metrics are published. + public AsyncMessagePump(MessagePumpOptions options, IEnumerable queueConfigs, ILogger logger = null, IMetrics metrics = null) + { + ValidateOptions(options); + ValidateQueueConfigs(queueConfigs); + + foreach (var queueConfig in queueConfigs) + { + var queueManager = new QueueManager(options.ConnectionString, queueConfig.QueueName, true, options.QueueClientOptions, options.BlobClientOptions); + var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(options.ConnectionString, queueConfig.PoisonQueueName); + _queueManagers.TryAdd(queueConfig.QueueName, (queueConfig, queueManager, poisonQueueManager, DateTime.MinValue, TimeSpan.Zero)); + } + + _queueNames = new RoundRobinList(queueConfigs.Select(config => config.QueueName)); + _logger = logger; + _metrics = metrics ?? TurnOffMetrics(); + _mesagePumpOptions = options; + + InitDefaultActions(); + RandomizeRoundRobinStart(); + } + + /// + /// Initializes a new instance of the class. + /// + /// /// The queue manager. /// The poison queue manager. - /// The number of concurrent tasks. /// The visibility timeout. /// The maximum dequeue count. /// The logger. /// The system where metrics are published. - public AsyncMessagePump(QueueManager queueManager, QueueManager poisonQueueManager, int concurrentTasks = 25, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + internal AsyncMessagePump(MessagePumpOptions options, QueueManager queueManager, QueueManager poisonQueueManager = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + : this(options, new[] { (queueManager, poisonQueueManager, visibilityTimeout, maxDequeueCount) }, logger, metrics) { - if (concurrentTasks < 1) throw new ArgumentOutOfRangeException("Number of concurrent tasks must be greather than zero", nameof(concurrentTasks)); - if (maxDequeueCount < 1) throw new ArgumentOutOfRangeException("Number of retries must be greather than zero", nameof(maxDequeueCount)); - - _queueManager = queueManager ?? throw new ArgumentNullException(nameof(queueManager)); - _poisonQueueManager = poisonQueueManager; - _concurrentTasks = concurrentTasks; - _visibilityTimeout = visibilityTimeout; - _maxDequeueCount = maxDequeueCount; + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The configuration options for each queue to be monitored. + /// The logger. + /// The system where metrics are published. + internal AsyncMessagePump(MessagePumpOptions options, IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs, ILogger logger = null, IMetrics metrics = null) + { + ValidateOptions(options); + ValidateQueueConfigs(queueConfigs); + + foreach (var queueConfig in queueConfigs) + { + var queueName = queueConfig.QueueManager.QueueName; + var poisonQueueName = queueConfig.PoisonQueueManager?.QueueName; + var config = new QueueConfig(queueName, poisonQueueName, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); + _queueManagers.TryAdd(queueName, (config, queueConfig.QueueManager, queueConfig.PoisonQueueManager, DateTime.MinValue, TimeSpan.Zero)); + } + + _queueNames = new RoundRobinList(queueConfigs.Select(config => config.QueueManager.QueueName)); _logger = logger; _metrics = metrics ?? TurnOffMetrics(); + _mesagePumpOptions = options; InitDefaultActions(); + RandomizeRoundRobinStart(); } #endregion @@ -129,10 +189,7 @@ public AsyncMessagePump(QueueManager queueManager, QueueManager poisonQueueManag public async Task StartAsync(CancellationToken cancellationToken) { if (OnMessage == null) throw new ArgumentNullException(nameof(OnMessage)); - - _logger?.LogTrace("AsyncMessagePump starting..."); - await ProcessMessagesAsync(_visibilityTimeout, cancellationToken).ConfigureAwait(false); - _logger?.LogTrace("AsyncMessagePump stopping..."); + await ProcessMessagesAsync(cancellationToken).ConfigureAwait(false); } #endregion @@ -141,8 +198,7 @@ public async Task StartAsync(CancellationToken cancellationToken) private void InitDefaultActions() { - OnError = (message, exception, isPoison) => _logger?.LogError(exception, "An error occured when processing a message"); - OnQueueEmpty = (cancellationToken) => Task.Delay(1500, cancellationToken).Wait(); + OnError = (queueName, message, exception, isPoison) => _logger?.LogError(exception, "An error occured when processing a message in {queueName}", queueName); } private IMetrics TurnOffMetrics() @@ -156,62 +212,58 @@ private IMetrics TurnOffMetrics() return metricsTurnedOff.Build(); } - private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, CancellationToken cancellationToken) + private void ValidateOptions(MessagePumpOptions options) + { + if (options == null) throw new ArgumentNullException(nameof(options)); + if (string.IsNullOrEmpty(options.ConnectionString)) throw new ArgumentNullException(nameof(options.ConnectionString)); + if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); + } + + private void ValidateQueueConfigs(IEnumerable queueConfigs) + { + if (queueConfigs == null || !queueConfigs.Any()) throw new ArgumentNullException(nameof(queueConfigs), "You must specify the configuration options for at least one queue"); + + var dequeueCountTooSmall = queueConfigs.Where(config => config.MaxDequeueCount < 1); + if (dequeueCountTooSmall.Any()) + { + var misConfiguredQueues = string.Join(", ", dequeueCountTooSmall.Select(config => config.QueueName)); + throw new ArgumentOutOfRangeException(nameof(queueConfigs), $"Number of retries is misconfigured for the following queues: {misConfiguredQueues}"); + } + } + + private void ValidateQueueConfigs(IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs) + { + if (queueConfigs == null || !queueConfigs.Any()) throw new ArgumentNullException(nameof(queueConfigs), "You must specify the configuration options for at least one queue"); + + if (queueConfigs.Any(config => config.QueueManager == null)) + { + throw new ArgumentNullException(nameof(queueConfigs), "At least one of the specified queue managers is null."); + } + + var dequeueCountTooSmall = queueConfigs.Where(config => config.MaxDequeueCount < 1); + if (dequeueCountTooSmall.Any()) + { + var misConfiguredQueues = string.Join(", ", dequeueCountTooSmall.Select(config => config.QueueManager.QueueName)); + throw new ArgumentOutOfRangeException(nameof(queueConfigs), $"Number of retries is misconfigured for the following queues: {misConfiguredQueues}"); + } + } + + private async Task ProcessMessagesAsync(CancellationToken cancellationToken) { var runningTasks = new ConcurrentDictionary(); - var semaphore = new SemaphoreSlim(_concurrentTasks, _concurrentTasks); - var queuedMessages = new ConcurrentQueue(); + var semaphore = new SemaphoreSlim(_mesagePumpOptions.ConcurrentTasks, _mesagePumpOptions.ConcurrentTasks); + var queuedMessages = new ConcurrentQueue<(string QueueName, CloudMessage Message)>(); // Define the task that fetches messages from the Azure queue RecurrentCancellableTask.StartNew( async () => { - // Fetch messages from the Azure queue when the number of items in the concurrent queue falls below an "acceptable" level. - if (!cancellationToken.IsCancellationRequested && queuedMessages.Count <= _concurrentTasks / 2) + // Fetch messages from Azure when the number of items in the concurrent queue falls below an "acceptable" level. + if (!cancellationToken.IsCancellationRequested && queuedMessages.Count <= _mesagePumpOptions.ConcurrentTasks / 2) { - IEnumerable messages = null; - using (_metrics.Measure.Timer.Time(Metrics.MessagesFetchingTimer)) - { - try - { - messages = await _queueManager.GetMessagesAsync(_concurrentTasks, visibilityTimeout, cancellationToken).ConfigureAwait(false); - } - catch (TaskCanceledException) - { - // The message pump is shutting down. - // This exception can be safely ignored. - } - catch (Exception e) - { - _logger?.LogError(e.GetBaseException(), "An error occured while fetching messages from the Azure queue. The error was caught and ignored."); - } - } - - if (messages == null) return; - - if (messages.Any()) + await foreach (var message in FetchMessages(cancellationToken)) { - var messagesCount = messages.Count(); - _logger?.LogTrace("Fetched {messagesCount} message(s) from the queue.", messagesCount); - - foreach (var message in messages) - { - queuedMessages.Enqueue(message); - } - } - else - { - _logger?.LogTrace("The queue is empty, no messages fetched."); - try - { - // The queue is empty - _metrics.Measure.Counter.Increment(Metrics.QueueEmptyCounter); - OnQueueEmpty?.Invoke(cancellationToken); - } - catch (Exception e) - { - _logger?.LogError(e.GetBaseException(), "An error occured when handling an empty queue. The error was caught and ignored."); - } + queuedMessages.Enqueue(message); } } }, @@ -219,24 +271,40 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio cancellationToken, TaskCreationOptions.LongRunning); - // Define the task that checks how many messages are queued in the Azure queue + // Define the task that checks how many messages are queued in Azure RecurrentCancellableTask.StartNew( async () => { - try - { - var count = await _queueManager.GetApproximateMessageCountAsync(cancellationToken).ConfigureAwait(false); - _metrics.Measure.Gauge.SetValue(Metrics.QueuedCloudMessagesGauge, count); - } - catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) - { - // The message pump is shutting down. - // This exception can be safely ignored. - } - catch (Exception e) + var count = 0; + foreach (var kvp in _queueManagers) { - _logger?.LogError(e.GetBaseException(), "An error occured while checking how many message are waiting in the Azure queue. The error was caught and ignored."); + var queueName = kvp.Key; + (var queueConfig, var queueManager, var poisonQueueManager, var lastFetched, var fetchDelay) = kvp.Value; + + try + { + var properties = await queueManager.GetPropertiesAsync(cancellationToken).ConfigureAwait(false); + + count += properties.ApproximateMessagesCount; + } + catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) + { + // The message pump is shutting down. + // This exception can be safely ignored. + } + catch (RequestFailedException rfe) when (rfe.ErrorCode == "QueueNotFound") + { + // The queue has been deleted + _queueNames.Remove(queueName); + _queueManagers.TryRemove(queueName, out _); + } + catch (Exception e) + { + _logger?.LogError(e.GetBaseException(), "An error occured while checking how many message are waiting in Azure. The error was caught and ignored."); + } } + + _metrics.Measure.Gauge.SetValue(Metrics.QueuedCloudMessagesGauge, count); }, TimeSpan.FromMilliseconds(5000), cancellationToken, @@ -276,50 +344,55 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio { var messageProcessed = false; - using (_metrics.Measure.Timer.Time(Metrics.MessageProcessingTimer)) + if (queuedMessages.TryDequeue(out (string QueueName, CloudMessage Message) result)) { - queuedMessages.TryDequeue(out CloudMessage message); - - if (message != null) + if (_queueManagers.TryGetValue(result.QueueName, out (QueueConfig Config, QueueManager QueueManager, QueueManager PoisonQueueManager, DateTime LastFetched, TimeSpan FetchDelay) queueInfo)) { - try + using (_metrics.Measure.Timer.Time(Metrics.MessageProcessingTimer)) { - // Process the message - OnMessage?.Invoke(message, cancellationToken); - - // Delete the processed message from the queue - // PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue even when the message pump is shutting down - await _queueManager.DeleteMessageAsync(message, CancellationToken.None).ConfigureAwait(false); - } - catch (Exception ex) - { - var isPoison = message.DequeueCount >= _maxDequeueCount; - try { - OnError?.Invoke(message, ex, isPoison); + // Process the message + OnMessage?.Invoke(result.QueueName, result.Message, cancellationToken); + + // Delete the processed message from the queue + // PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue even when the message pump is shutting down + await queueInfo.QueueManager.DeleteMessageAsync(result.Message, CancellationToken.None).ConfigureAwait(false); } - catch (Exception e) + catch (Exception ex) { - _logger?.LogError(e.GetBaseException(), "An error occured when handling an exception. The error was caught and ignored."); - } + var isPoison = result.Message.DequeueCount >= queueInfo.Config.MaxDequeueCount; - if (isPoison) - { - // PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue and moved to poison queue even when the message pump is shutting down - if (_poisonQueueManager != null) + try { - message.Metadata["PoisonExceptionMessage"] = ex.GetBaseException().Message; - message.Metadata["PoisonExceptionDetails"] = ex.GetBaseException().ToString(); - - await _poisonQueueManager.AddMessageAsync(message.Content, message.Metadata, null, null, CancellationToken.None).ConfigureAwait(false); + OnError?.Invoke(result.QueueName, result.Message, ex, isPoison); + } + catch (Exception e) + { + _logger?.LogError(e.GetBaseException(), "An error occured when handling an exception for {queueName}. The error was caught and ignored.", result.QueueName); } - await _queueManager.DeleteMessageAsync(message, CancellationToken.None).ConfigureAwait(false); + if (isPoison) + { + // PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue and moved to poison queue even when the message pump is shutting down + if (queueInfo.PoisonQueueManager != null) + { + result.Message.Metadata["PoisonExceptionMessage"] = ex.GetBaseException().Message; + result.Message.Metadata["PoisonExceptionDetails"] = ex.GetBaseException().ToString(); + + await queueInfo.PoisonQueueManager.AddMessageAsync(result.Message.Content, result.Message.Metadata, null, null, CancellationToken.None).ConfigureAwait(false); + } + + await queueInfo.QueueManager.DeleteMessageAsync(result.Message, CancellationToken.None).ConfigureAwait(false); + } } - } - messageProcessed = true; + messageProcessed = true; + } + } + else + { + _queueNames.Remove(result.QueueName); } } @@ -352,7 +425,113 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio // Task pump has been canceled, wait for the currently running tasks to complete await Task.WhenAll(runningTasks.Values).UntilCancelled().ConfigureAwait(false); } - } - #endregion + private async IAsyncEnumerable<(string QueueName, CloudMessage Message)> FetchMessages([EnumeratorCancellation] CancellationToken cancellationToken) + { + var messageCount = 0; + + if (string.IsNullOrEmpty(_queueNames.Current)) _queueNames.Reset(); + var originalQueue = _queueNames.Current; + + using (_metrics.Measure.Timer.Time(Metrics.MessagesFetchingTimer)) + { + do + { + var queueName = _queueNames.MoveToNextItem(); + + if (_queueManagers.TryGetValue(queueName, out (QueueConfig Config, QueueManager QueueManager, QueueManager PoisonQueueManager, DateTime LastFetched, TimeSpan FetchDelay) queueInfo)) + { + if (!cancellationToken.IsCancellationRequested && queueInfo.LastFetched.Add(queueInfo.FetchDelay) < DateTime.UtcNow) + { + IEnumerable messages = null; + + try + { + messages = await queueInfo.QueueManager.GetMessagesAsync(_mesagePumpOptions.ConcurrentTasks, queueInfo.Config.VisibilityTimeout, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) + { + // The message pump is shutting down. + // This exception can be safely ignored. + } + catch (RequestFailedException rfe) when (rfe.ErrorCode == "QueueNotFound") + { + // The queue has been deleted + _queueNames.Remove(queueName); + _queueManagers.TryRemove(queueName, out _); + } + catch (Exception e) + { + _logger?.LogError(e.GetBaseException(), "An error occured while fetching messages from {queueName}. The error was caught and ignored.", queueName); + } + + if (messages != null && messages.Any()) + { + var messagesCount = messages.Count(); + _logger?.LogTrace("Fetched {messagesCount} message(s) in {queueName}.", messagesCount, queueName); + + foreach (var message in messages) + { + Interlocked.Increment(ref messageCount); + yield return (queueName, message); + } + + // Reset the Fetch delay to zero to indicate that we can fetch more messages from this queue as soon as possible + _queueManagers[queueName] = (queueInfo.Config, queueInfo.QueueManager, queueInfo.PoisonQueueManager, DateTime.UtcNow, TimeSpan.Zero); + } + else + { + _logger?.LogTrace("There are no messages in {queueName}.", queueName); + _metrics.Measure.Counter.Increment(Metrics.QueueEmptyCounter); + + // Set a "resonable" fetch delay to ensure we don't query an empty queue too often + var delay = queueInfo.FetchDelay.Add(TimeSpan.FromSeconds(5)); + if (delay.TotalSeconds > 15) delay = TimeSpan.FromSeconds(15); + + _queueManagers[queueName] = (queueInfo.Config, queueInfo.QueueManager, queueInfo.PoisonQueueManager, DateTime.UtcNow, delay); + } + } + } + else + { + _queueNames.Remove(queueName); + } + } + + // Stop when we either retrieved the desired number of messages OR we have looped through all the queues + while (messageCount < (_mesagePumpOptions.ConcurrentTasks * 2) && originalQueue != _queueNames.Current); + } + + if (messageCount == 0) + { + _logger?.LogTrace("All tenant queues are empty, no messages fetched."); + try + { + // All queues are empty + _metrics.Measure.Counter.Increment(Metrics.AllQueuesEmptyCounter); + OnEmpty?.Invoke(cancellationToken); + } + catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) + { + // The message pump is shutting down. + // This exception can be safely ignored. + } + catch (Exception e) + { + _logger?.LogError(e.GetBaseException(), "An error occured when handling empty queues. The error was caught and ignored."); + } + } + } + + private void RandomizeRoundRobinStart() + { + if (_queueNames.Current == null) + { + var randomIndex = RandomGenerator.Instance.GetInt32(0, _queueNames.Count); + _queueNames.ResetTo(randomIndex); + } + } + + #endregion + } } diff --git a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs index 4ee1bee..64b2a39 100644 --- a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs @@ -1,13 +1,9 @@ using App.Metrics; -using Microsoft.Extensions.DependencyModel; using Microsoft.Extensions.Logging; using Picton.Managers; -using Picton.Messaging.Messages; +using Picton.Messaging.Utilities; using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -15,18 +11,17 @@ namespace Picton.Messaging { /// /// High performance message processor (also known as a message "pump") for Azure storage queues. - /// Designed to monitor an Azure storage queue and process the message as quickly and efficiently as possible. + /// Designed to monitor either a single queue or a fixed list of queues and process messages as + /// quickly and efficiently as possible. /// public class AsyncMessagePumpWithHandlers { #region FIELDS - private static readonly ReaderWriterLockSlim _lock = new ReaderWriterLockSlim(); - private static IDictionary _messageHandlers; - private readonly ILogger _logger; private readonly AsyncMessagePump _messagePump; + private readonly ILogger _logger; #endregion @@ -43,97 +38,98 @@ public class AsyncMessagePumpWithHandlers /// /// When isPoison is set to true, you should copy this message to a poison queue because it will be deleted from the original queue. /// - public Action OnError - { - get { return _messagePump.OnError; } - set { _messagePump.OnError = value; } - } + public Action OnError { get; set; } /// - /// Gets or sets the logic to execute when queue is empty. + /// Gets or sets the logic to execute when all queues are empty. /// /// /// - /// OnQueueEmpty = cancellationToken => Task.Delay(2500, cancellationToken).Wait(); + /// OnEmpty = cancellationToken => Task.Delay(2500, cancellationToken).Wait(); /// /// /// - /// If this property is not set, the default logic is to pause for 2 seconds. + /// If this property is not set, the default logic is to do nothing. /// - public Action OnQueueEmpty - { - get { return _messagePump.OnQueueEmpty; } - set { _messagePump.OnQueueEmpty = value; } - } + public Action OnEmpty { get; set; } #endregion - #region CONSTRUCTOR + #region CONSTRUCTORS /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// /// A connection string includes the authentication information required for your application to access data in an Azure Storage account at runtime. /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. /// /// Name of the queue. - /// The number of concurrent tasks. - /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poinson' messages. + /// + /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. /// The visibility timeout. /// The maximum dequeue count. /// The logger. /// The system where metrics are published. [ExcludeFromCodeCoverage] - public AsyncMessagePumpWithHandlers(string connectionString, string queueName, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) - : this(new QueueManager(connectionString, queueName), string.IsNullOrEmpty(poisonQueueName) ? null : new QueueManager(connectionString, poisonQueueName), concurrentTasks, visibilityTimeout, maxDequeueCount, logger, metrics) + public AsyncMessagePumpWithHandlers(MessagePumpOptions options, string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + : this(options, new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount), logger, metrics) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// + /// The logger. + /// The system where metrics are published. + [ExcludeFromCodeCoverage] + public AsyncMessagePumpWithHandlers(MessagePumpOptions options, QueueConfig queueConfig, ILogger logger = null, IMetrics metrics = null) + : this(options, new[] { queueConfig }, logger, metrics) { } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// + /// + /// The configuration options for each queue to be monitored. + /// The logger. + /// The system where metrics are published. + public AsyncMessagePumpWithHandlers(MessagePumpOptions options, IEnumerable queueConfigs, ILogger logger = null, IMetrics metrics = null) + { + _messagePump = new AsyncMessagePump(options, queueConfigs, logger, metrics); + _logger = logger; + } + + /// + /// Initializes a new instance of the class. + /// + /// /// The queue manager. /// The poison queue manager. - /// The number of concurrent tasks. /// The visibility timeout. /// The maximum dequeue count. /// The logger. /// The system where metrics are published. - public AsyncMessagePumpWithHandlers(QueueManager queueManager, QueueManager poisonQueueManager, int concurrentTasks = 25, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + internal AsyncMessagePumpWithHandlers(MessagePumpOptions options, QueueManager queueManager, QueueManager poisonQueueManager = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + : this(options, new[] { (queueManager, poisonQueueManager, visibilityTimeout, maxDequeueCount) }, logger, metrics) { - _logger = logger; - - _messagePump = new AsyncMessagePump(queueManager, poisonQueueManager, concurrentTasks, visibilityTimeout, maxDequeueCount, logger, metrics) - { - OnMessage = (message, cancellationToken) => - { - var contentType = message.Content.GetType(); - - if (!_messageHandlers.TryGetValue(contentType, out Type[] handlers)) - { - throw new Exception($"Received a message of type {contentType.FullName} but could not find a class implementing IMessageHandler<{contentType.FullName}>"); - } - - foreach (var handlerType in handlers) - { - object handler = null; - if (handlerType.GetConstructor(new[] { typeof(ILogger) }) != null) - { - handler = Activator.CreateInstance(handlerType, new[] { (object)logger }); - } - else - { - handler = Activator.CreateInstance(handlerType); - } - - var handlerMethod = handlerType.GetMethod("Handle", new[] { contentType }); - handlerMethod.Invoke(handler, new[] { message.Content }); - } - } - }; + } - DiscoverMessageHandlersIfNecessary(logger); + /// + /// Initializes a new instance of the class. + /// + /// + /// The configuration options for each queue to be monitored. + /// The logger. + /// The system where metrics are published. + internal AsyncMessagePumpWithHandlers(MessagePumpOptions options, IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs, ILogger logger = null, IMetrics metrics = null) + { + _messageHandlers = MessageHandlersDiscoverer.GetMessageHandlers(logger); + _messagePump = new AsyncMessagePump(options, queueConfigs, logger, metrics); + _logger = logger; } #endregion @@ -148,114 +144,35 @@ public AsyncMessagePumpWithHandlers(QueueManager queueManager, QueueManager pois /// A representing the asynchronous operation. public Task StartAsync(CancellationToken cancellationToken) { - return _messagePump.StartAsync(cancellationToken); - } - - #endregion - - #region PRIVATE METHODS - - private static void DiscoverMessageHandlersIfNecessary(ILogger logger) - { - try + _messagePump.OnEmpty = OnEmpty; + _messagePump.OnError = OnError; + _messagePump.OnMessage = (queueName, message, cancellationToken) => { - _lock.EnterUpgradeableReadLock(); + var contentType = message.Content.GetType(); - if (_messageHandlers == null) + if (!_messageHandlers.TryGetValue(contentType, out Type[] handlers)) { - try - { - _lock.EnterWriteLock(); + throw new Exception($"Received a message of type {contentType.FullName} but could not find a class implementing IMessageHandler<{contentType.FullName}>"); + } - if (_messageHandlers == null) - { - _messageHandlers = GetMessageHandlers(null); - } + foreach (var handlerType in handlers) + { + object handler = null; + if (handlerType.GetConstructor(new[] { typeof(ILogger) }) != null) + { + handler = Activator.CreateInstance(handlerType, new[] { (object)_logger }); } - finally + else { - if (_lock.IsWriteLockHeld) _lock.ExitWriteLock(); + handler = Activator.CreateInstance(handlerType); } - } - } - finally - { - if (_lock.IsUpgradeableReadLockHeld) _lock.ExitUpgradeableReadLock(); - } - } - - private static IDictionary GetMessageHandlers(ILogger logger) - { - logger?.LogTrace("Discovering message handlers."); - var assemblies = GetLocalAssemblies(); - - var assembliesCount = assemblies.Length; - if (assembliesCount == 0) logger?.LogTrace($"Did not find any local assembly."); - else if (assembliesCount == 1) logger?.LogTrace("Found 1 local assembly."); - else logger?.LogTrace($"Found {assemblies.Count()} local assemblies."); - - var typesWithMessageHandlerInterfaces = assemblies - .SelectMany(x => x.GetTypes()) - .Where(t => !t.GetTypeInfo().IsInterface) - .Select(type => new - { - Type = type, - MessageTypes = type - .GetInterfaces() - .Where(i => i.GetTypeInfo().IsGenericType) - .Where(i => i.GetGenericTypeDefinition() == typeof(IMessageHandler<>)) - .SelectMany(i => i.GetGenericArguments()) - }) - .Where(t => t.MessageTypes != null && t.MessageTypes.Any()) - .ToArray(); - - var classesCount = typesWithMessageHandlerInterfaces.Length; - if (classesCount == 0) logger?.LogTrace("Did not find any class implementing the 'IMessageHandler' interface."); - else if (classesCount == 1) logger?.LogTrace("Found 1 class implementing the 'IMessageHandler' interface."); - else logger?.LogTrace($"Found {typesWithMessageHandlerInterfaces.Count()} classes implementing the 'IMessageHandler' interface."); - - var oneTypePerMessageHandler = typesWithMessageHandlerInterfaces - .SelectMany(t => t.MessageTypes, (t, messageType) => - new - { - t.Type, - MessageType = messageType - }) - .ToArray(); - - var messageHandlers = oneTypePerMessageHandler - .GroupBy(h => h.MessageType) - .ToDictionary(group => group.Key, group => group.Select(t => t.Type) - .ToArray()); - - return messageHandlers; - } - - private static Assembly[] GetLocalAssemblies() - { - var dependencies = DependencyContext.Default.RuntimeLibraries; - - var assemblies = new List(); - foreach (var library in dependencies) - { - if (IsCandidateLibrary(library)) - { - var assembly = Assembly.Load(new AssemblyName(library.Name)); - assemblies.Add(assembly); + var handlerMethod = handlerType.GetMethod("Handle", new[] { contentType }); + handlerMethod.Invoke(handler, new[] { message.Content }); } - } - - return assemblies.ToArray(); - } + }; - private static bool IsCandidateLibrary(RuntimeLibrary library) - { - return !library.Name.StartsWith("Microsoft.", StringComparison.OrdinalIgnoreCase) && - !library.Name.StartsWith("System.", StringComparison.OrdinalIgnoreCase) && - !library.Name.StartsWith("NetStandard.", StringComparison.OrdinalIgnoreCase) && - !string.Equals(library.Type, "package", StringComparison.OrdinalIgnoreCase) && - !string.Equals(library.Type, "referenceassembly", StringComparison.OrdinalIgnoreCase); + return _messagePump.StartAsync(cancellationToken); } #endregion diff --git a/Source/Picton.Messaging/MessagePumpOptions.cs b/Source/Picton.Messaging/MessagePumpOptions.cs new file mode 100644 index 0000000..e003e52 --- /dev/null +++ b/Source/Picton.Messaging/MessagePumpOptions.cs @@ -0,0 +1,42 @@ +using Azure.Storage.Blobs; +using Azure.Storage.Queues; +using System; + +namespace Picton.Messaging +{ + public record MessagePumpOptions + { + public MessagePumpOptions(string connectionString, int concurrentTasks, QueueClientOptions queueClientOptions = null, BlobClientOptions blobClientOptions = null) + { + ConnectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString)); + ConcurrentTasks = concurrentTasks; + QueueClientOptions = queueClientOptions; + BlobClientOptions = blobClientOptions; + } + + /// + /// Gets or sets the connection string which includes the authentication information required for your application to access data in an Azure Storage account at runtime. + /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. + /// + public string ConnectionString { get; set; } + + /// + /// Gets or sets the number of concurrent tasks. In other words: the number of messages that can be processed at a time. + /// + public int ConcurrentTasks { get; set; } = 25; + + /// + /// Gets or sets the optional client options that define the transport + /// pipeline policies for authentication, retries, etc., that are applied + /// to every request to the queue. + /// + public QueueClientOptions QueueClientOptions { get; set; } = null; + + /// + /// Gets or sets the optional client options that define the transport + /// pipeline policies for authentication, retries, etc., that are applied + /// to every request to the blob storage. + /// + public BlobClientOptions BlobClientOptions { get; set; } = null; + } +} diff --git a/Source/Picton.Messaging/QueueConfig.cs b/Source/Picton.Messaging/QueueConfig.cs new file mode 100644 index 0000000..2487968 --- /dev/null +++ b/Source/Picton.Messaging/QueueConfig.cs @@ -0,0 +1,23 @@ +using System; + +namespace Picton.Messaging +{ + public record QueueConfig + { + public QueueConfig(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3) + { + QueueName = queueName ?? throw new ArgumentNullException(nameof(queueName)); + PoisonQueueName = poisonQueueName; + VisibilityTimeout = visibilityTimeout; + MaxDequeueCount = maxDequeueCount; + } + + public string QueueName { get; set; } + + public string PoisonQueueName { get; set; } = null; + + public TimeSpan? VisibilityTimeout { get; set; } = null; + + public int MaxDequeueCount { get; set; } = 3; + } +} From f64d8073bf05d19883e1249d118c85de165cc297 Mon Sep 17 00:00:00 2001 From: jericho Date: Fri, 5 Jan 2024 12:09:20 -0500 Subject: [PATCH 14/28] Static methods to instantiate new message pumps rather than constructors --- Source/Picton.Messaging/AsyncMessagePump.cs | 141 +++++++----------- .../AsyncMessagePumpWithHandlers.cs | 117 +++++++++------ 2 files changed, 132 insertions(+), 126 deletions(-) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index bfda366..22c72e9 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -71,39 +71,6 @@ public class AsyncMessagePump #region CONSTRUCTOR - /// - /// Initializes a new instance of the class. - /// - /// - /// A connection string includes the authentication information required for your application to access data in an Azure Storage account at runtime. - /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. - /// - /// Name of the queue. - /// - /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. - /// The visibility timeout. - /// The maximum dequeue count. - /// The logger. - /// The system where metrics are published. - [ExcludeFromCodeCoverage] - public AsyncMessagePump(MessagePumpOptions options, string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) - : this(options, new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount), logger, metrics) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// - /// - /// The logger. - /// The system where metrics are published. - [ExcludeFromCodeCoverage] - public AsyncMessagePump(MessagePumpOptions options, QueueConfig queueConfig, ILogger logger = null, IMetrics metrics = null) - : this(options, new[] { queueConfig }, logger, metrics) - { - } - /// /// Initializes a new instance of the class. /// @@ -111,19 +78,17 @@ public AsyncMessagePump(MessagePumpOptions options, QueueConfig queueConfig, ILo /// The configuration options for each queue to be monitored. /// The logger. /// The system where metrics are published. - public AsyncMessagePump(MessagePumpOptions options, IEnumerable queueConfigs, ILogger logger = null, IMetrics metrics = null) + internal AsyncMessagePump(MessagePumpOptions options, IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs, ILogger logger = null, IMetrics metrics = null) { - ValidateOptions(options); - ValidateQueueConfigs(queueConfigs); - foreach (var queueConfig in queueConfigs) { - var queueManager = new QueueManager(options.ConnectionString, queueConfig.QueueName, true, options.QueueClientOptions, options.BlobClientOptions); - var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(options.ConnectionString, queueConfig.PoisonQueueName); - _queueManagers.TryAdd(queueConfig.QueueName, (queueConfig, queueManager, poisonQueueManager, DateTime.MinValue, TimeSpan.Zero)); + var queueName = queueConfig.QueueManager.QueueName; + var poisonQueueName = queueConfig.PoisonQueueManager?.QueueName; + var config = new QueueConfig(queueName, poisonQueueName, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); + _queueManagers.TryAdd(queueName, (config, queueConfig.QueueManager, queueConfig.PoisonQueueManager, DateTime.MinValue, TimeSpan.Zero)); } - _queueNames = new RoundRobinList(queueConfigs.Select(config => config.QueueName)); + _queueNames = new RoundRobinList(queueConfigs.Select(config => config.QueueManager.QueueName)); _logger = logger; _metrics = metrics ?? TurnOffMetrics(); _mesagePumpOptions = options; @@ -132,48 +97,70 @@ public AsyncMessagePump(MessagePumpOptions options, IEnumerable que RandomizeRoundRobinStart(); } + #endregion + + #region STATIC METHODS + /// - /// Initializes a new instance of the class. + /// Returns a message pump that will process messages in a single Azure storage queue. /// /// - /// The queue manager. - /// The poison queue manager. + /// Name of the queue. + /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. /// The visibility timeout. /// The maximum dequeue count. /// The logger. /// The system where metrics are published. - internal AsyncMessagePump(MessagePumpOptions options, QueueManager queueManager, QueueManager poisonQueueManager = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) - : this(options, new[] { (queueManager, poisonQueueManager, visibilityTimeout, maxDequeueCount) }, logger, metrics) + /// The message pump. + public AsyncMessagePump ForSingleQueue(MessagePumpOptions options, string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + { + var queueConfig = new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount); + + ValidateOptions(options); + ValidateQueueConfig(queueConfig); + + return ForMultipleQueues(options, new[] { queueConfig }, logger, metrics); + } + + /// + /// Returns a message pump that will process messages in a single Azure storage queue. + /// + /// + /// + /// The logger. + /// The system where metrics are published. + /// The message pump. + public AsyncMessagePump ForSingleQueue(MessagePumpOptions options, QueueConfig queueConfig, ILogger logger = null, IMetrics metrics = null) { + ValidateOptions(options); + ValidateQueueConfig(queueConfig); + + return ForMultipleQueues(options, new[] { queueConfig }, logger, metrics); } /// - /// Initializes a new instance of the class. + /// Returns a message pump that will process messages in multiple Azure storage queues. /// /// - /// The configuration options for each queue to be monitored. + /// /// The logger. /// The system where metrics are published. - internal AsyncMessagePump(MessagePumpOptions options, IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs, ILogger logger = null, IMetrics metrics = null) + /// The message pump. + public AsyncMessagePump ForMultipleQueues(MessagePumpOptions options, IEnumerable queueConfigs, ILogger logger = null, IMetrics metrics = null) { ValidateOptions(options); ValidateQueueConfigs(queueConfigs); - foreach (var queueConfig in queueConfigs) - { - var queueName = queueConfig.QueueManager.QueueName; - var poisonQueueName = queueConfig.PoisonQueueManager?.QueueName; - var config = new QueueConfig(queueName, poisonQueueName, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); - _queueManagers.TryAdd(queueName, (config, queueConfig.QueueManager, queueConfig.PoisonQueueManager, DateTime.MinValue, TimeSpan.Zero)); - } - - _queueNames = new RoundRobinList(queueConfigs.Select(config => config.QueueManager.QueueName)); - _logger = logger; - _metrics = metrics ?? TurnOffMetrics(); - _mesagePumpOptions = options; + var configs = queueConfigs + .Select(queueConfig => + { + var queueManager = new QueueManager(options.ConnectionString, queueConfig.QueueName, true, options.QueueClientOptions, options.BlobClientOptions); + var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(options.ConnectionString, queueConfig.PoisonQueueName); + return (queueManager, poisonQueueManager, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); + }) + .ToArray(); - InitDefaultActions(); - RandomizeRoundRobinStart(); + return new AsyncMessagePump(options, configs, logger, metrics); } #endregion @@ -219,32 +206,18 @@ private void ValidateOptions(MessagePumpOptions options) if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); } - private void ValidateQueueConfigs(IEnumerable queueConfigs) + private void ValidateQueueConfig(QueueConfig queueConfig) { - if (queueConfigs == null || !queueConfigs.Any()) throw new ArgumentNullException(nameof(queueConfigs), "You must specify the configuration options for at least one queue"); - - var dequeueCountTooSmall = queueConfigs.Where(config => config.MaxDequeueCount < 1); - if (dequeueCountTooSmall.Any()) - { - var misConfiguredQueues = string.Join(", ", dequeueCountTooSmall.Select(config => config.QueueName)); - throw new ArgumentOutOfRangeException(nameof(queueConfigs), $"Number of retries is misconfigured for the following queues: {misConfiguredQueues}"); - } + if (queueConfig == null) throw new ArgumentNullException(nameof(queueConfig)); + if (string.IsNullOrEmpty(queueConfig.QueueName)) throw new ArgumentNullException(nameof(queueConfig.QueueName)); + if (queueConfig.MaxDequeueCount < 1) throw new ArgumentOutOfRangeException(nameof(queueConfig.MaxDequeueCount), $"Number of retries for {queueConfig.QueueName} must be greater than zero."); } - private void ValidateQueueConfigs(IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs) + private void ValidateQueueConfigs(IEnumerable queueConfigs) { - if (queueConfigs == null || !queueConfigs.Any()) throw new ArgumentNullException(nameof(queueConfigs), "You must specify the configuration options for at least one queue"); - - if (queueConfigs.Any(config => config.QueueManager == null)) - { - throw new ArgumentNullException(nameof(queueConfigs), "At least one of the specified queue managers is null."); - } - - var dequeueCountTooSmall = queueConfigs.Where(config => config.MaxDequeueCount < 1); - if (dequeueCountTooSmall.Any()) + foreach (var queueConfig in queueConfigs) { - var misConfiguredQueues = string.Join(", ", dequeueCountTooSmall.Select(config => config.QueueManager.QueueName)); - throw new ArgumentOutOfRangeException(nameof(queueConfigs), $"Number of retries is misconfigured for the following queues: {misConfiguredQueues}"); + ValidateQueueConfig(queueConfig); } } diff --git a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs index 64b2a39..a6f2ca4 100644 --- a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs @@ -4,6 +4,7 @@ using Picton.Messaging.Utilities; using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -58,78 +59,84 @@ public class AsyncMessagePumpWithHandlers #region CONSTRUCTORS /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - /// - /// A connection string includes the authentication information required for your application to access data in an Azure Storage account at runtime. - /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. - /// - /// Name of the queue. /// - /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. - /// The visibility timeout. - /// The maximum dequeue count. + /// The configuration options for each queue to be monitored. /// The logger. /// The system where metrics are published. - [ExcludeFromCodeCoverage] - public AsyncMessagePumpWithHandlers(MessagePumpOptions options, string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) - : this(options, new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount), logger, metrics) + internal AsyncMessagePumpWithHandlers(MessagePumpOptions options, IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs, ILogger logger = null, IMetrics metrics = null) { - } + _messageHandlers = MessageHandlersDiscoverer.GetMessageHandlers(logger); + _messagePump = new AsyncMessagePump(options, queueConfigs, logger, metrics); + _logger = logger; - /// - /// Initializes a new instance of the class. - /// - /// - /// - /// The logger. - /// The system where metrics are published. - [ExcludeFromCodeCoverage] - public AsyncMessagePumpWithHandlers(MessagePumpOptions options, QueueConfig queueConfig, ILogger logger = null, IMetrics metrics = null) - : this(options, new[] { queueConfig }, logger, metrics) - { } + #endregion + + #region STATIC METHODS + /// - /// Initializes a new instance of the class. + /// Returns a message pump that will process messages in a single Azure storage queue. /// /// - /// The configuration options for each queue to be monitored. + /// Name of the queue. + /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. + /// The visibility timeout. + /// The maximum dequeue count. /// The logger. /// The system where metrics are published. - public AsyncMessagePumpWithHandlers(MessagePumpOptions options, IEnumerable queueConfigs, ILogger logger = null, IMetrics metrics = null) + /// The message pump. + public AsyncMessagePumpWithHandlers ForSingleQueue(MessagePumpOptions options, string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) { - _messagePump = new AsyncMessagePump(options, queueConfigs, logger, metrics); - _logger = logger; + var queueConfig = new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount); + + ValidateOptions(options); + ValidateQueueConfig(queueConfig); + + return ForMultipleQueues(options, new[] { queueConfig }, logger, metrics); } /// - /// Initializes a new instance of the class. + /// Returns a message pump that will process messages in a single Azure storage queue. /// /// - /// The queue manager. - /// The poison queue manager. - /// The visibility timeout. - /// The maximum dequeue count. + /// /// The logger. /// The system where metrics are published. - internal AsyncMessagePumpWithHandlers(MessagePumpOptions options, QueueManager queueManager, QueueManager poisonQueueManager = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) - : this(options, new[] { (queueManager, poisonQueueManager, visibilityTimeout, maxDequeueCount) }, logger, metrics) + /// The message pump. + public AsyncMessagePumpWithHandlers ForSingleQueue(MessagePumpOptions options, QueueConfig queueConfig, ILogger logger = null, IMetrics metrics = null) { + ValidateOptions(options); + ValidateQueueConfig(queueConfig); + + return ForMultipleQueues(options, new[] { queueConfig }, logger, metrics); } /// - /// Initializes a new instance of the class. + /// Returns a message pump that will process messages in multiple Azure storage queues. /// /// - /// The configuration options for each queue to be monitored. + /// /// The logger. /// The system where metrics are published. - internal AsyncMessagePumpWithHandlers(MessagePumpOptions options, IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs, ILogger logger = null, IMetrics metrics = null) + /// The message pump. + public AsyncMessagePumpWithHandlers ForMultipleQueues(MessagePumpOptions options, IEnumerable queueConfigs, ILogger logger = null, IMetrics metrics = null) { - _messageHandlers = MessageHandlersDiscoverer.GetMessageHandlers(logger); - _messagePump = new AsyncMessagePump(options, queueConfigs, logger, metrics); - _logger = logger; + ValidateOptions(options); + ValidateQueueConfigs(queueConfigs); + + var configs = queueConfigs + .Select(queueConfig => + { + var queueManager = new QueueManager(options.ConnectionString, queueConfig.QueueName, true, options.QueueClientOptions, options.BlobClientOptions); + var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(options.ConnectionString, queueConfig.PoisonQueueName); + return (queueManager, poisonQueueManager, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); + }) + .ToArray(); + + return new AsyncMessagePumpWithHandlers(options, configs, logger, metrics); } #endregion @@ -176,5 +183,31 @@ public Task StartAsync(CancellationToken cancellationToken) } #endregion + + #region PRIVATE METHODS + + private void ValidateOptions(MessagePumpOptions options) + { + if (options == null) throw new ArgumentNullException(nameof(options)); + if (string.IsNullOrEmpty(options.ConnectionString)) throw new ArgumentNullException(nameof(options.ConnectionString)); + if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); + } + + private void ValidateQueueConfig(QueueConfig queueConfig) + { + if (queueConfig == null) throw new ArgumentNullException(nameof(queueConfig)); + if (string.IsNullOrEmpty(queueConfig.QueueName)) throw new ArgumentNullException(nameof(queueConfig.QueueName)); + if (queueConfig.MaxDequeueCount < 1) throw new ArgumentOutOfRangeException(nameof(queueConfig.MaxDequeueCount), $"Number of retries for {queueConfig.QueueName} must be greater than zero."); + } + + private void ValidateQueueConfigs(IEnumerable queueConfigs) + { + foreach (var queueConfig in queueConfigs) + { + ValidateQueueConfig(queueConfig); + } + } + + #endregion } } From eee51adb90895a62d0f9210ec25dfd2c194b293f Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 7 Jan 2024 11:58:00 -0500 Subject: [PATCH 15/28] (GH-33) Handle multiple queues --- README.md | 28 +- .../TestsRunner.cs | 57 ++- .../AsyncMessagePumpTests.cs | 78 ++-- .../Picton.Messaging.UnitTests/MockUtils.cs | 4 + Source/Picton.Messaging/AsyncMessagePump.cs | 171 +++----- .../AsyncMessagePumpWithHandlers.cs | 80 +--- .../AsyncMultiTenantMessagePump.cs | 389 ++---------------- ...AsyncMultiTenantMessagePumpWithHandlers.cs | 209 +++------- .../Utilities/RoundRobinList.cs | 72 +++- 9 files changed, 322 insertions(+), 766 deletions(-) diff --git a/README.md b/README.md index 9d04d24..6733894 100644 --- a/README.md +++ b/README.md @@ -108,16 +108,17 @@ namespace WorkerRole1 private async Task RunAsync(CancellationToken cancellationToken) { var connectionString = "<-- insert connection string for your Azure account -->"; - var queueName = "<-- insert the name of your Azure queue -->"; + var concurrentTask = 10; // <-- this is the max number of messages that can be processed at a time // Configure the message pump - var messagePump = new AsyncMessagePump(connectionString, queueName, 10, null, TimeSpan.FromMinutes(1), 3) + var options = new MessagePumpOptions(connectionString, concurrentTasks); + var messagePump = new AsyncMessagePump(options) { - OnMessage = (message, cancellationToken) => + OnMessage = (queueName, message, cancellationToken) => { // This is where you insert your custom logic to process a message }, - OnError = (message, exception, isPoison) => + OnError = (queueName, message, exception, isPoison) => { // Insert your custom error handling @@ -127,18 +128,23 @@ namespace WorkerRole1 // this parameter indicates whether this message has exceeded the maximum // number of retries. // - // When you have configured the "poison queue name" and this parameter is - // "true", the message is automatically copied to the poison queue and - // removed from the original queue. + // When you have configured the "poison queue name" for the given queue and + // this parameter is "true", the message is automatically copied to the poison + // queue and removed from the original queue. // - // If you have not configured the "poison queue name" and this parameter is - // "true", the message is automatically removed from the original queue and - // you are responsible for storing the message. If you don't, this mesage - // will be lost. + // If you have not configured the "poison queue name" for the given queue and + // this parameter is "true", the message is automatically removed from the + // original queue and you are responsible for storing the message. If you don't, + // this mesage will be lost. // ========================================================================== } }; + // Replace the following samples with the queues you want to monitor + messagePump.AddQueue("myfirstqueue", "myfirstqueue-poison", TimeSpan.FromMinutes(1), 3); + messagePump.AddQueue("mysecondqueue", "mysecondqueue-poison", TimeSpan.FromMinutes(1), 3); + messagePump.AddQueue("mythirdqueue", "mythirdqueue-poison", TimeSpan.FromMinutes(1), 3); + // Start the message pump await messagePump.StartAsync(cancellationToken); } diff --git a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs index 10afb66..7079c42 100644 --- a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs +++ b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs @@ -72,13 +72,12 @@ public async Task RunAsync() { var connectionString = "UseDevelopmentStorage=true"; var queueName = "myqueue"; - var numberOfMessages = 25; - var numberOfTenants = 5; + var concurrentTasks = 5; // Run the integration tests - await RunAsyncMessagePumpTests(connectionString, queueName, numberOfMessages, metrics, cts.Token).ConfigureAwait(false); - await RunAsyncMessagePumpWithHandlersTests(connectionString, queueName, numberOfMessages, metrics, cts.Token).ConfigureAwait(false); - await RunMultiTenantAsyncMessagePumpTests(connectionString, queueName, numberOfTenants, numberOfMessages, metrics, cts.Token).ConfigureAwait(false); + await RunAsyncMessagePumpTests(connectionString, queueName, concurrentTasks, 25, metrics, cts.Token).ConfigureAwait(false); + await RunAsyncMessagePumpWithHandlersTests(connectionString, queueName, concurrentTasks, 25, metrics, cts.Token).ConfigureAwait(false); + await RunMultiTenantAsyncMessagePumpTests(connectionString, queueName, concurrentTasks, [6, 12, 18, 24], metrics, cts.Token).ConfigureAwait(false); } // Prompt user to press a key in order to allow reading the log in the console @@ -93,7 +92,7 @@ public async Task RunAsync() return await Task.FromResult(resultCode); } - private async Task RunAsyncMessagePumpTests(string connectionString, string queueName, int numberOfMessages, IMetrics metrics, CancellationToken cancellationToken) + private async Task RunAsyncMessagePumpTests(string connectionString, string queueName, int concurrentTasks, int numberOfMessages, IMetrics metrics, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) return; @@ -109,18 +108,17 @@ private async Task RunAsyncMessagePumpTests(string connectionString, string queu await queueManager.AddMessageAsync($"Hello world {i}").ConfigureAwait(false); } - // Process the messages - Stopwatch sw = null; - // Configure the message pump - var options = new MessagePumpOptions(connectionString, 10, null, null); - var messagePump = new AsyncMessagePump(options, queueName, null, TimeSpan.FromMinutes(1), 3, _logger, metrics) + Stopwatch sw = null; + var options = new MessagePumpOptions(connectionString, concurrentTasks, null, null); + var messagePump = new AsyncMessagePump(options, _logger, metrics) { OnMessage = (queueName, message, cancellationToken) => { _logger.LogInformation(message.Content.ToString()); } }; + messagePump.AddQueue(queueName, null, TimeSpan.FromMinutes(1), 3); // Stop the message pump when the queue is empty. var cts = new CancellationTokenSource(); @@ -143,7 +141,7 @@ private async Task RunAsyncMessagePumpTests(string connectionString, string queu _logger.LogInformation($"\tDone in {sw.Elapsed.ToDurationString()}"); } - private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, string queueName, int numberOfMessages, IMetrics metrics, CancellationToken cancellationToken) + private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, string queueName, int concurrentTasks, int numberOfMessages, IMetrics metrics, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) return; @@ -159,22 +157,23 @@ private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, await queueManager.AddMessageAsync(new MyMessage { MessageContent = $"Hello world {i}" }).ConfigureAwait(false); } - // Process the messages - Stopwatch sw = null; - // Configure the message pump + Stopwatch sw = null; var cts = new CancellationTokenSource(); - var options = new MessagePumpOptions(connectionString, 10, null, null); - var messagePump = new AsyncMessagePumpWithHandlers(options, queueName, null, TimeSpan.FromMinutes(1), 3, _logger, metrics); - messagePump.OnEmpty = cancellationToken => + var options = new MessagePumpOptions(connectionString, concurrentTasks, null, null); + var messagePump = new AsyncMessagePumpWithHandlers(options, _logger, metrics) { - // Stop the timer - if (sw.IsRunning) sw.Stop(); + OnEmpty = cancellationToken => + { + // Stop the timer + if (sw.IsRunning) sw.Stop(); - // Stop the message pump - _logger.LogDebug("Asking the message pump with handlers to stop..."); - cts.Cancel(); + // Stop the message pump + _logger.LogDebug("Asking the message pump with handlers to stop..."); + cts.Cancel(); + } }; + messagePump.AddQueue(queueName, null, TimeSpan.FromMinutes(1), 3); // Start the message pump sw = Stopwatch.StartNew(); @@ -185,7 +184,7 @@ private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, _logger.LogInformation($"\tDone in {sw.Elapsed.ToDurationString()}"); } - private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, string queueNamePrefix, int numberOfTenants, int numberOfMessages, IMetrics metrics, CancellationToken cancellationToken) + private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, string queueNamePrefix, int concurrentTasks, int[] numberOfMessagesForTenant, IMetrics metrics, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) return; @@ -193,14 +192,13 @@ private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, _logger.LogInformation("Testing AsyncMultiTenantMessagePump..."); // Add messages to the tenant queues - for (int i = 0; i < numberOfTenants; i++) + for (int i = 0; i < numberOfMessagesForTenant.Length; i++) { var queueManager = new QueueManager(connectionString, $"{queueNamePrefix}{i:00}"); await queueManager.ClearAsync().ConfigureAwait(false); - var numberOfMessagesForThisTenant = numberOfMessages * (i + 1); // Each tenant receives a different number of messages - for (var j = 0; j < numberOfMessagesForThisTenant; j++) + for (var j = 0; j < numberOfMessagesForTenant[i]; j++) { - await queueManager.AddMessageAsync($"Hello world {j} to tenant {i:00}").ConfigureAwait(false); + await queueManager.AddMessageAsync($"Hello world {j:00} to tenant {i:00}").ConfigureAwait(false); } } @@ -209,7 +207,8 @@ private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, // Configure the message pump var cts = new CancellationTokenSource(); - var messagePump = new AsyncMultiTenantMessagePump(connectionString, queueNamePrefix, 10, null, TimeSpan.FromMinutes(1), 3, null, null, _logger, metrics) + var options = new MessagePumpOptions(connectionString, concurrentTasks, null, null); + var messagePump = new AsyncMultiTenantMessagePump(options, queueNamePrefix, TimeSpan.FromMinutes(1), 3, _logger, metrics) { OnMessage = (tenantId, message, cancellationToken) => { diff --git a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs index 6e63d7f..72c66d9 100644 --- a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs +++ b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs @@ -14,46 +14,55 @@ namespace Picton.Messaging.UnitTests public class AsyncMessagePumpTests { [Fact] - public void Null_cloudQueue_throws() + public void Throws_when_options_is_null() { + // Arrange + var options = (MessagePumpOptions)null; + + //Act + Should.Throw(() => new AsyncMessagePump(options)); + } + + [Fact] + public void Throws_when_zero_queues() + { + // Arrange + var cts = new CancellationTokenSource(); + var options = new MessagePumpOptions("bogus connection string", 1); + var messagePump = new AsyncMessagePump(options); + + // Act + Should.ThrowAsync(() => messagePump.StartAsync(cts.Token)); - Should.Throw(() => - { - var messagePump = new AsyncMessagePump(options, (QueueManager)null); - }); } [Fact] - public void Number_of_concurrent_tasks_too_small_throws() + public void Throws_when_number_of_concurrent_tasks_too_small() { - Should.Throw(() => - { - var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); - var mockQueueClient = MockUtils.GetMockQueueClient(); - var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, false); - var options = new MessagePumpOptions("bogus connection string", 0); + // Arrange + var options = new MessagePumpOptions("bogus connection string", 0); - var messagePump = new AsyncMessagePump(options, queueManager); - }); + //Act + Should.Throw(() => new AsyncMessagePump(options)); } [Fact] - public void DequeueCount_too_small_throws() + public void Throws_when_DequeueCount_too_small() { - Should.Throw(() => - { - var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); - var mockQueueClient = MockUtils.GetMockQueueClient(); - var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, false); - var options = new MessagePumpOptions("bogus connection string", 0); + // Arrange + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); + var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient, false); + var options = new MessagePumpOptions("bogus connection string", 1); + var messagePump = new AsyncMessagePump(options); - var messagePump = new AsyncMessagePump(options, queueManager, maxDequeueCount: 0); - }); + // Act + Should.Throw(() => messagePump.AddQueue(queueManager, null, null, 0)); } [Fact] - public void Start_without_OnMessage_throws() + public void Throws_when_OnMessage_not_set() { // Arrange var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); @@ -63,7 +72,8 @@ public void Start_without_OnMessage_throws() var cts = new CancellationTokenSource(); - var messagePump = new AsyncMessagePump(options, queueManager); + var messagePump = new AsyncMessagePump(options); + messagePump.AddQueue(queueManager, null, null, 3); // Act Should.ThrowAsync(() => messagePump.StartAsync(cts.Token)); @@ -96,7 +106,7 @@ public async Task No_message_processed_when_queue_is_empty() var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(options, queueManager) + var messagePump = new AsyncMessagePump(options) { OnMessage = (queueName, message, cancellationToken) => { @@ -112,6 +122,7 @@ public async Task No_message_processed_when_queue_is_empty() cts.Cancel(); } }; + messagePump.AddQueue(queueManager, null, null, 3); // Act await messagePump.StartAsync(cts.Token); @@ -184,7 +195,7 @@ public async Task Message_processed() var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(options, queueManager) + var messagePump = new AsyncMessagePump(options) { OnMessage = (queueName, message, cancellationToken) => { @@ -207,6 +218,7 @@ public async Task Message_processed() cts.Cancel(); } }; + messagePump.AddQueue(queueManager, null, null, 3); // Act await messagePump.StartAsync(cts.Token); @@ -276,7 +288,7 @@ public async Task Poison_message_is_rejected() var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(options, queueManager) + var messagePump = new AsyncMessagePump(options) { OnMessage = (queueName, message, cancellationToken) => { @@ -293,6 +305,7 @@ public async Task Poison_message_is_rejected() cts.Cancel(); } }; + messagePump.AddQueue(queueManager, null, null, retries); // Act await messagePump.StartAsync(cts.Token); @@ -373,7 +386,7 @@ public async Task Poison_message_is_moved() var poisonQueueManager = new QueueManager(mockBlobContainerClient, mockPoisonQueueClient); var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(options, queueManager, poisonQueueManager) + var messagePump = new AsyncMessagePump(options) { OnMessage = (queueName, message, cancellationToken) => { @@ -390,6 +403,7 @@ public async Task Poison_message_is_moved() cts.Cancel(); } }; + messagePump.AddQueue(queueManager, poisonQueueManager, null, retries); // Act await messagePump.StartAsync(cts.Token); @@ -431,7 +445,7 @@ public async Task Exceptions_in_OnEmpty_are_ignored() var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(options, queueManager) + var messagePump = new AsyncMessagePump(options) { OnMessage = (queueName, message, cancellationToken) => { @@ -459,6 +473,7 @@ public async Task Exceptions_in_OnEmpty_are_ignored() cts.Cancel(); } }; + messagePump.AddQueue(queueManager, null, null, 3); // Act await messagePump.StartAsync(cts.Token); @@ -524,7 +539,7 @@ public async Task Exceptions_in_OnError_are_ignored() var queueManager = new QueueManager(mockBlobContainerClient, mockQueueClient); var options = new MessagePumpOptions("bogus connection string", 1); - var messagePump = new AsyncMessagePump(options, queueManager) + var messagePump = new AsyncMessagePump(options) { OnMessage = (queueName, message, cancellationToken) => { @@ -542,6 +557,7 @@ public async Task Exceptions_in_OnError_are_ignored() cts.Cancel(); } }; + messagePump.AddQueue(queueManager, null, null, 3); // Act await messagePump.StartAsync(cts.Token); diff --git a/Source/Picton.Messaging.UnitTests/MockUtils.cs b/Source/Picton.Messaging.UnitTests/MockUtils.cs index 6d1f1c7..6e2d751 100644 --- a/Source/Picton.Messaging.UnitTests/MockUtils.cs +++ b/Source/Picton.Messaging.UnitTests/MockUtils.cs @@ -64,6 +64,10 @@ internal static QueueClient GetMockQueueClient(string queueName = "myqueue") var mockQueueStorageUri = new Uri(QUEUE_STORAGE_URL + queueName); var mockQueueClient = Substitute.For(); + mockQueueClient + .Name + .Returns(queueName); + mockQueueClient .Uri .Returns(mockQueueStorageUri); diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 22c72e9..60ed63c 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -15,7 +15,7 @@ namespace Picton.Messaging { /// /// High performance message processor (also known as a message "pump") for Azure storage queues. - /// Designed to monitor either a single queue or a fixed list of queues and process messages as + /// Designed to monitor either a single queue or a list of queues and process messages as /// quickly and efficiently as possible. /// public class AsyncMessagePump @@ -23,9 +23,9 @@ public class AsyncMessagePump #region FIELDS private readonly ConcurrentDictionary _queueManagers = new ConcurrentDictionary(); - private readonly RoundRobinList _queueNames; + private readonly RoundRobinList _queueNames = new RoundRobinList(Enumerable.Empty()); - private readonly MessagePumpOptions _mesagePumpOptions; + private readonly MessagePumpOptions _messagePumpOptions; private readonly ILogger _logger; private readonly IMetrics _metrics; @@ -72,101 +72,52 @@ public class AsyncMessagePump #region CONSTRUCTOR /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - /// - /// The configuration options for each queue to be monitored. + /// Options for the mesage pump. /// The logger. /// The system where metrics are published. - internal AsyncMessagePump(MessagePumpOptions options, IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs, ILogger logger = null, IMetrics metrics = null) + public AsyncMessagePump(MessagePumpOptions options, ILogger logger = null, IMetrics metrics = null) { - foreach (var queueConfig in queueConfigs) - { - var queueName = queueConfig.QueueManager.QueueName; - var poisonQueueName = queueConfig.PoisonQueueManager?.QueueName; - var config = new QueueConfig(queueName, poisonQueueName, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); - _queueManagers.TryAdd(queueName, (config, queueConfig.QueueManager, queueConfig.PoisonQueueManager, DateTime.MinValue, TimeSpan.Zero)); - } + if (options == null) throw new ArgumentNullException(nameof(options)); + if (string.IsNullOrEmpty(options.ConnectionString)) throw new ArgumentNullException(nameof(options.ConnectionString)); + if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); - _queueNames = new RoundRobinList(queueConfigs.Select(config => config.QueueManager.QueueName)); + _messagePumpOptions = options; _logger = logger; _metrics = metrics ?? TurnOffMetrics(); - _mesagePumpOptions = options; InitDefaultActions(); - RandomizeRoundRobinStart(); } #endregion - #region STATIC METHODS + #region PUBLIC METHODS - /// - /// Returns a message pump that will process messages in a single Azure storage queue. - /// - /// - /// Name of the queue. - /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. - /// The visibility timeout. - /// The maximum dequeue count. - /// The logger. - /// The system where metrics are published. - /// The message pump. - public AsyncMessagePump ForSingleQueue(MessagePumpOptions options, string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + public void AddQueue(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3) { - var queueConfig = new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount); - - ValidateOptions(options); - ValidateQueueConfig(queueConfig); - - return ForMultipleQueues(options, new[] { queueConfig }, logger, metrics); + AddQueue(new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount)); } - /// - /// Returns a message pump that will process messages in a single Azure storage queue. - /// - /// - /// - /// The logger. - /// The system where metrics are published. - /// The message pump. - public AsyncMessagePump ForSingleQueue(MessagePumpOptions options, QueueConfig queueConfig, ILogger logger = null, IMetrics metrics = null) + public void AddQueue(QueueConfig queueConfig) { - ValidateOptions(options); - ValidateQueueConfig(queueConfig); + if (string.IsNullOrEmpty(queueConfig.QueueName)) throw new ArgumentNullException(nameof(queueConfig.QueueName)); + if (queueConfig.MaxDequeueCount < 1) throw new ArgumentOutOfRangeException(nameof(queueConfig.MaxDequeueCount), "Number of retries must be greater than zero."); + + var queueManager = new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.QueueName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); + var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.PoisonQueueName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); - return ForMultipleQueues(options, new[] { queueConfig }, logger, metrics); + AddQueue(queueManager, poisonQueueManager, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); } - /// - /// Returns a message pump that will process messages in multiple Azure storage queues. - /// - /// - /// - /// The logger. - /// The system where metrics are published. - /// The message pump. - public AsyncMessagePump ForMultipleQueues(MessagePumpOptions options, IEnumerable queueConfigs, ILogger logger = null, IMetrics metrics = null) + public void RemoveQueue(string queueName) { - ValidateOptions(options); - ValidateQueueConfigs(queueConfigs); - - var configs = queueConfigs - .Select(queueConfig => - { - var queueManager = new QueueManager(options.ConnectionString, queueConfig.QueueName, true, options.QueueClientOptions, options.BlobClientOptions); - var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(options.ConnectionString, queueConfig.PoisonQueueName); - return (queueManager, poisonQueueManager, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); - }) - .ToArray(); + // Do not remove from _queuManagers because there could messages still in the memory queue that need to be processed + //_queueManagers.TryRemove(queueName, out _); - return new AsyncMessagePump(options, configs, logger, metrics); + _queueNames.RemoveItem(queueName); } - #endregion - - #region PUBLIC METHODS - /// /// Starts the message pump. /// @@ -183,6 +134,22 @@ public async Task StartAsync(CancellationToken cancellationToken) #region PRIVATE METHODS + // This internal method is primarily for unit testing purposes. It allows me to inject mocked queue managers + internal void AddQueue(QueueManager queueManager, QueueManager poisonQueueManager, TimeSpan? visibilityTimeout, int maxDequeueCount) + { + if (queueManager == null) throw new ArgumentNullException(nameof(queueManager)); + if (string.IsNullOrEmpty(queueManager.QueueName)) throw new ArgumentNullException(nameof(queueManager.QueueName)); + if (maxDequeueCount < 1) throw new ArgumentOutOfRangeException(nameof(maxDequeueCount), "Number of retries must be greater than zero."); + + var queueConfig = new QueueConfig(queueManager.QueueName, poisonQueueManager?.QueueName, visibilityTimeout, maxDequeueCount); + + _queueManagers.AddOrUpdate( + queueManager.QueueName, + (queueName) => (queueConfig, queueManager, poisonQueueManager, DateTime.MinValue, TimeSpan.Zero), + (queueName, oldConfig) => (queueConfig, queueManager, poisonQueueManager, oldConfig.LastFetched, oldConfig.FetchDelay)); + _queueNames.AddItem(queueManager.QueueName); + } + private void InitDefaultActions() { OnError = (queueName, message, exception, isPoison) => _logger?.LogError(exception, "An error occured when processing a message in {queueName}", queueName); @@ -199,32 +166,10 @@ private IMetrics TurnOffMetrics() return metricsTurnedOff.Build(); } - private void ValidateOptions(MessagePumpOptions options) - { - if (options == null) throw new ArgumentNullException(nameof(options)); - if (string.IsNullOrEmpty(options.ConnectionString)) throw new ArgumentNullException(nameof(options.ConnectionString)); - if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); - } - - private void ValidateQueueConfig(QueueConfig queueConfig) - { - if (queueConfig == null) throw new ArgumentNullException(nameof(queueConfig)); - if (string.IsNullOrEmpty(queueConfig.QueueName)) throw new ArgumentNullException(nameof(queueConfig.QueueName)); - if (queueConfig.MaxDequeueCount < 1) throw new ArgumentOutOfRangeException(nameof(queueConfig.MaxDequeueCount), $"Number of retries for {queueConfig.QueueName} must be greater than zero."); - } - - private void ValidateQueueConfigs(IEnumerable queueConfigs) - { - foreach (var queueConfig in queueConfigs) - { - ValidateQueueConfig(queueConfig); - } - } - private async Task ProcessMessagesAsync(CancellationToken cancellationToken) { var runningTasks = new ConcurrentDictionary(); - var semaphore = new SemaphoreSlim(_mesagePumpOptions.ConcurrentTasks, _mesagePumpOptions.ConcurrentTasks); + var semaphore = new SemaphoreSlim(_messagePumpOptions.ConcurrentTasks, _messagePumpOptions.ConcurrentTasks); var queuedMessages = new ConcurrentQueue<(string QueueName, CloudMessage Message)>(); // Define the task that fetches messages from the Azure queue @@ -232,7 +177,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) async () => { // Fetch messages from Azure when the number of items in the concurrent queue falls below an "acceptable" level. - if (!cancellationToken.IsCancellationRequested && queuedMessages.Count <= _mesagePumpOptions.ConcurrentTasks / 2) + if (!cancellationToken.IsCancellationRequested && queuedMessages.Count <= _messagePumpOptions.ConcurrentTasks / 2) { await foreach (var message in FetchMessages(cancellationToken)) { @@ -268,8 +213,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) catch (RequestFailedException rfe) when (rfe.ErrorCode == "QueueNotFound") { // The queue has been deleted - _queueNames.Remove(queueName); - _queueManagers.TryRemove(queueName, out _); + RemoveQueue(queueName); } catch (Exception e) { @@ -352,6 +296,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) { result.Message.Metadata["PoisonExceptionMessage"] = ex.GetBaseException().Message; result.Message.Metadata["PoisonExceptionDetails"] = ex.GetBaseException().ToString(); + result.Message.Metadata["PoisonOriginalQueue"] = queueInfo.QueueManager.QueueName; await queueInfo.PoisonQueueManager.AddMessageAsync(result.Message.Content, result.Message.Metadata, null, null, CancellationToken.None).ConfigureAwait(false); } @@ -365,7 +310,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) } else { - _queueNames.Remove(result.QueueName); + _queueNames.RemoveItem(result.QueueName); } } @@ -403,7 +348,12 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) { var messageCount = 0; - if (string.IsNullOrEmpty(_queueNames.Current)) _queueNames.Reset(); + if (_queueNames.Count == 0) + { + _logger?.LogTrace("There are no tenant queues being monitored. Therefore no messages could be fetched."); + yield break; + } + var originalQueue = _queueNames.Current; using (_metrics.Measure.Timer.Time(Metrics.MessagesFetchingTimer)) @@ -411,6 +361,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) do { var queueName = _queueNames.MoveToNextItem(); + originalQueue ??= queueName; // This is important because originalQueue will be null the very first time we fetch messages if (_queueManagers.TryGetValue(queueName, out (QueueConfig Config, QueueManager QueueManager, QueueManager PoisonQueueManager, DateTime LastFetched, TimeSpan FetchDelay) queueInfo)) { @@ -420,7 +371,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) try { - messages = await queueInfo.QueueManager.GetMessagesAsync(_mesagePumpOptions.ConcurrentTasks, queueInfo.Config.VisibilityTimeout, cancellationToken).ConfigureAwait(false); + messages = await queueInfo.QueueManager.GetMessagesAsync(_messagePumpOptions.ConcurrentTasks, queueInfo.Config.VisibilityTimeout, cancellationToken).ConfigureAwait(false); } catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) { @@ -430,8 +381,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) catch (RequestFailedException rfe) when (rfe.ErrorCode == "QueueNotFound") { // The queue has been deleted - _queueNames.Remove(queueName); - _queueManagers.TryRemove(queueName, out _); + RemoveQueue(queueName); } catch (Exception e) { @@ -467,12 +417,12 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) } else { - _queueNames.Remove(queueName); + _queueNames.RemoveItem(queueName); } } // Stop when we either retrieved the desired number of messages OR we have looped through all the queues - while (messageCount < (_mesagePumpOptions.ConcurrentTasks * 2) && originalQueue != _queueNames.Current); + while (messageCount < (_messagePumpOptions.ConcurrentTasks * 2) && originalQueue != _queueNames.Next); } if (messageCount == 0) @@ -496,15 +446,6 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) } } - private void RandomizeRoundRobinStart() - { - if (_queueNames.Current == null) - { - var randomIndex = RandomGenerator.Instance.GetInt32(0, _queueNames.Count); - _queueNames.ResetTo(randomIndex); - } - } - #endregion } } diff --git a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs index a6f2ca4..cebfcf7 100644 --- a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs @@ -4,7 +4,6 @@ using Picton.Messaging.Utilities; using System; using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -61,88 +60,35 @@ public class AsyncMessagePumpWithHandlers /// /// Initializes a new instance of the class. /// - /// - /// The configuration options for each queue to be monitored. + /// Options for the mesage pump. /// The logger. /// The system where metrics are published. - internal AsyncMessagePumpWithHandlers(MessagePumpOptions options, IEnumerable<(QueueManager QueueManager, QueueManager PoisonQueueManager, TimeSpan? VisibilityTimeout, int MaxDequeueCount)> queueConfigs, ILogger logger = null, IMetrics metrics = null) + public AsyncMessagePumpWithHandlers(MessagePumpOptions options, ILogger logger = null, IMetrics metrics = null) { _messageHandlers = MessageHandlersDiscoverer.GetMessageHandlers(logger); - _messagePump = new AsyncMessagePump(options, queueConfigs, logger, metrics); + _messagePump = new AsyncMessagePump(options, logger, metrics); _logger = logger; - } #endregion - #region STATIC METHODS + #region PUBLIC METHODS - /// - /// Returns a message pump that will process messages in a single Azure storage queue. - /// - /// - /// Name of the queue. - /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. - /// The visibility timeout. - /// The maximum dequeue count. - /// The logger. - /// The system where metrics are published. - /// The message pump. - public AsyncMessagePumpWithHandlers ForSingleQueue(MessagePumpOptions options, string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + public void AddQueue(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3) { - var queueConfig = new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount); - - ValidateOptions(options); - ValidateQueueConfig(queueConfig); - - return ForMultipleQueues(options, new[] { queueConfig }, logger, metrics); + _messagePump.AddQueue(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount); } - /// - /// Returns a message pump that will process messages in a single Azure storage queue. - /// - /// - /// - /// The logger. - /// The system where metrics are published. - /// The message pump. - public AsyncMessagePumpWithHandlers ForSingleQueue(MessagePumpOptions options, QueueConfig queueConfig, ILogger logger = null, IMetrics metrics = null) + public void AddQueue(QueueConfig queueConfig) { - ValidateOptions(options); - ValidateQueueConfig(queueConfig); - - return ForMultipleQueues(options, new[] { queueConfig }, logger, metrics); + _messagePump.AddQueue(queueConfig); } - /// - /// Returns a message pump that will process messages in multiple Azure storage queues. - /// - /// - /// - /// The logger. - /// The system where metrics are published. - /// The message pump. - public AsyncMessagePumpWithHandlers ForMultipleQueues(MessagePumpOptions options, IEnumerable queueConfigs, ILogger logger = null, IMetrics metrics = null) + public void RemoveQueue(string queueName) { - ValidateOptions(options); - ValidateQueueConfigs(queueConfigs); - - var configs = queueConfigs - .Select(queueConfig => - { - var queueManager = new QueueManager(options.ConnectionString, queueConfig.QueueName, true, options.QueueClientOptions, options.BlobClientOptions); - var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(options.ConnectionString, queueConfig.PoisonQueueName); - return (queueManager, poisonQueueManager, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); - }) - .ToArray(); - - return new AsyncMessagePumpWithHandlers(options, configs, logger, metrics); + _messagePump.RemoveQueue(queueName); } - #endregion - - #region PUBLIC METHODS - /// /// Starts the message pump. /// @@ -186,6 +132,12 @@ public Task StartAsync(CancellationToken cancellationToken) #region PRIVATE METHODS + // This internal method is primarily for unit testing purposes. It allows me to inject mocked queue managers + internal void AddQueue(QueueManager queueManager, QueueManager poisonQueueManager, TimeSpan? visibilityTimeout, int maxDequeueCount) + { + _messagePump.AddQueue(queueManager, poisonQueueManager, visibilityTimeout, maxDequeueCount); + } + private void ValidateOptions(MessagePumpOptions options) { if (options == null) throw new ArgumentNullException(nameof(options)); diff --git a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs index 8c7f3c9..4d5103f 100644 --- a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs @@ -1,17 +1,10 @@ using App.Metrics; using Azure; -using Azure.Storage.Blobs; using Azure.Storage.Queues; using Azure.Storage.Queues.Models; using Microsoft.Extensions.Logging; -using Picton.Interfaces; -using Picton.Managers; using Picton.Messaging.Utilities; using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -19,26 +12,29 @@ namespace Picton.Messaging { /// /// High performance message processor (also known as a message "pump") for Azure storage queues. + /// /// Designed to monitor multiple Azure storage queues that follow the following naming convention: - /// a common prefix followed by a unique tenant identifier. + /// a common prefix followed by a unique tenant identifier. For example, if the prefix is "myqueue", + /// this message pump will monitor queues such as "myqueue001", myqueue002" and "myqueueabc". + /// + /// Please note that the message pump specifically ignores queue that follow the following naming convention: + /// - the common prefix without a postfix. For example "myqueue". Notice the absence of a tenant identifier + /// after the "myqueue" part in the name. + /// - The common prefix followed by "-poison". For example "myqueue-poison". + /// + /// Furthermore, the list of queues matching the naming convention is refreshed at regular interval in order + /// to discover new tenant queues that might have been created in the Azure storage account. /// public class AsyncMultiTenantMessagePump { #region FIELDS - private readonly Func _queueManagerFactory; - - private readonly ConcurrentDictionary> _tenantQueueManagers = new ConcurrentDictionary>(); - private readonly RoundRobinList _tenantIds = new RoundRobinList(Array.Empty()); - - private readonly IQueueManager _poisonQueueManager; - private readonly string _connectionString; + private readonly MessagePumpOptions _messagePumpOptions; private readonly string _queueNamePrefix; - private readonly int _concurrentTasks; private readonly TimeSpan? _visibilityTimeout; private readonly int _maxDequeueCount; private readonly ILogger _logger; - private readonly IMetrics _metrics; + private readonly AsyncMessagePump _messagePump; #endregion @@ -66,7 +62,7 @@ public class AsyncMultiTenantMessagePump public Action OnError { get; set; } /// - /// Gets or sets the logic to execute when all tenant queues are empty. + /// Gets or sets the logic to execute when all queues are empty. /// /// /// @@ -85,49 +81,25 @@ public class AsyncMultiTenantMessagePump /// /// Initializes a new instance of the class. /// - /// - /// A connection string includes the authentication information required for your application to access data in an Azure Storage account at runtime. - /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. - /// - /// Queues name prefix. - /// The number of concurrent tasks. - /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. + /// Options for the mesage pump. + /// The common prefix in the naming convention. /// The visibility timeout. /// The maximum dequeue count. - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request to the queue. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request to the blob storage. - /// /// The logger. /// The system where metrics are published. - public AsyncMultiTenantMessagePump(string connectionString, string queueNamePrefix, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, QueueClientOptions queueClientOptions = null, BlobClientOptions blobClientOptions = null, ILogger logger = null, IMetrics metrics = null) + public AsyncMultiTenantMessagePump(MessagePumpOptions options, string queueNamePrefix, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) { - if (concurrentTasks < 1) throw new ArgumentOutOfRangeException("Number of concurrent tasks must be greather than zero", nameof(concurrentTasks)); - if (maxDequeueCount < 1) throw new ArgumentOutOfRangeException("Number of retries must be greather than zero", nameof(maxDequeueCount)); + if (options == null) throw new ArgumentNullException(nameof(options)); + if (string.IsNullOrEmpty(options.ConnectionString)) throw new ArgumentNullException(nameof(options.ConnectionString)); + if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); + if (string.IsNullOrEmpty(queueNamePrefix)) throw new ArgumentNullException(nameof(queueNamePrefix)); - _queueManagerFactory = (tenantId) => - { - var blobContainerClient = new BlobContainerClient(connectionString, $"{queueNamePrefix}{tenantId}-oversized-messages", blobClientOptions); - var queueClient = new QueueClient(connectionString, $"{queueNamePrefix}{tenantId}", queueClientOptions); - return new QueueManager(blobContainerClient, queueClient, true); - }; - - _connectionString = connectionString ?? throw new ArgumentNullException(connectionString); - _queueNamePrefix = queueNamePrefix ?? throw new ArgumentNullException(queueNamePrefix); - _concurrentTasks = concurrentTasks; - _poisonQueueManager = string.IsNullOrEmpty(poisonQueueName) ? null : new QueueManager(connectionString, poisonQueueName); + _messagePumpOptions = options; + _queueNamePrefix = queueNamePrefix; _visibilityTimeout = visibilityTimeout; _maxDequeueCount = maxDequeueCount; _logger = logger; - _metrics = metrics ?? TurnOffMetrics(); - - InitDefaultActions(); + _messagePump = new AsyncMessagePump(options, logger, metrics); } #endregion @@ -144,36 +116,9 @@ public async Task StartAsync(CancellationToken cancellationToken) { if (OnMessage == null) throw new ArgumentNullException(nameof(OnMessage)); - _logger?.LogTrace("AsyncMultiTenantMessagePump starting message pump..."); - await ProcessMessagesAsync(_visibilityTimeout, cancellationToken).ConfigureAwait(false); - _logger?.LogTrace("AsyncMultiTenantMessagePump stopping message pump..."); - } - - #endregion - - #region PRIVATE METHODS - - private void InitDefaultActions() - { - OnError = (tenantId, message, exception, isPoison) => _logger?.LogError(exception, "An error occured when processing a message for tenant {tenantId}", tenantId); - } - - private IMetrics TurnOffMetrics() - { - var metricsTurnedOff = new MetricsBuilder(); - metricsTurnedOff.Configuration.Configure(new MetricsOptions() - { - Enabled = false, - ReportingEnabled = false - }); - return metricsTurnedOff.Build(); - } - - private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, CancellationToken cancellationToken) - { - var runningTasks = new ConcurrentDictionary(); - var semaphore = new SemaphoreSlim(_concurrentTasks, _concurrentTasks); - var queuedMessages = new ConcurrentQueue<(string TenantId, CloudMessage Message)>(); + _messagePump.OnEmpty = OnEmpty; + _messagePump.OnError = (queueName, message, exception, isPoison) => OnError?.Invoke(queueName.TrimStart(_queueNamePrefix), message, exception, isPoison); + _messagePump.OnMessage = (queueName, message, cancellationToken) => OnMessage?.Invoke(queueName.TrimStart(_queueNamePrefix), message, cancellationToken); // Define the task that discovers queues that follow the naming convention RecurrentCancellableTask.StartNew( @@ -181,25 +126,25 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio { try { - var queueServiceClient = new QueueServiceClient(_connectionString); + var queueServiceClient = new QueueServiceClient(_messagePumpOptions.ConnectionString); var response = queueServiceClient.GetQueuesAsync(QueueTraits.None, _queueNamePrefix, cancellationToken); await foreach (Page queues in response.AsPages()) { foreach (var queue in queues.Values) { - if (!queue.Name.Equals(_queueNamePrefix, StringComparison.OrdinalIgnoreCase)) + if (!queue.Name.Equals(_queueNamePrefix, StringComparison.OrdinalIgnoreCase) && + !queue.Name.Equals($"{_queueNamePrefix}-poison", StringComparison.OrdinalIgnoreCase)) { - _tenantIds.Add(queue.Name.TrimStart(_queueNamePrefix)); + // AddQueue will make sure to add the queue only if it's not already in the round-robin list of queues. + _messagePump.AddQueue(queue.Name, $"{_queueNamePrefix}-poison", _visibilityTimeout, _maxDequeueCount); } } } - // Randomize where we start - if (_tenantIds.Current == null) - { - var randomIndex = RandomGenerator.Instance.GetInt32(0, _tenantIds.Count); - _tenantIds.ResetTo(randomIndex); - } + // Please note there is no need to remove queues that no longer exist from the message + // pump round-robin list. The reason is: message pump will get a RequestFailedException + // with ErrorCode == "QueueNotFound" next time the message pump tries to query those + // queues and it will automatically remove them at that time. } catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) { @@ -211,272 +156,14 @@ private async Task ProcessMessagesAsync(TimeSpan? visibilityTimeout, Cancellatio _logger?.LogError(e.GetBaseException(), "An error occured while fetching the Azure queues that match the naming convention. The error was caught and ignored."); } }, - TimeSpan.FromMilliseconds(15000), + TimeSpan.FromMilliseconds(30000), cancellationToken, TaskCreationOptions.LongRunning); // Brief pause to ensure the task defined above runs at least once before we start processing messages await Task.Delay(500, cancellationToken).ConfigureAwait(false); - // Define the task that fetches messages from the Azure queue - RecurrentCancellableTask.StartNew( - async () => - { - // Fetch messages from Azure when the number of items in the concurrent queue falls below an "acceptable" level. - if (!cancellationToken.IsCancellationRequested && queuedMessages.Count <= _concurrentTasks / 2) - { - await foreach (var message in FetchMessages(visibilityTimeout, cancellationToken)) - { - queuedMessages.Enqueue(message); - } - } - }, - TimeSpan.FromMilliseconds(500), - cancellationToken, - TaskCreationOptions.LongRunning); - - // Define the task that checks how many messages are queued in Azure - RecurrentCancellableTask.StartNew( - async () => - { - var count = 0; - foreach (var kvp in _tenantQueueManagers) - { - var tenantId = kvp.Key; - (var queueManager, var lastFetched, var fetchDelay) = kvp.Value.Value; - - try - { - var properties = await queueManager.GetPropertiesAsync(cancellationToken).ConfigureAwait(false); - - count += properties.ApproximateMessagesCount; - } - catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) - { - // The message pump is shutting down. - // This exception can be safely ignored. - } - catch (RequestFailedException rfe) when (rfe.ErrorCode == "QueueNotFound") - { - // The queue has been deleted - _tenantIds.Remove(tenantId); - _tenantQueueManagers.TryRemove(tenantId, out _); - } - catch (Exception e) - { - _logger?.LogError(e.GetBaseException(), "An error occured while checking how many message are waiting in Azure. The error was caught and ignored."); - } - } - - _metrics.Measure.Gauge.SetValue(Metrics.QueuedCloudMessagesGauge, count); - }, - TimeSpan.FromMilliseconds(5000), - cancellationToken, - TaskCreationOptions.LongRunning); - - // Define the task that checks how many messages are queued in memory - RecurrentCancellableTask.StartNew( - () => - { - try - { - _metrics.Measure.Gauge.SetValue(Metrics.QueuedMemoryMessagesGauge, queuedMessages.Count); - } - catch (Exception e) - { - _logger?.LogError(e.GetBaseException(), "An error occured while checking how many message are waiting in the memory queue. The error was caught and ignored."); - } - - return Task.CompletedTask; - }, - TimeSpan.FromMilliseconds(5000), - cancellationToken, - TaskCreationOptions.LongRunning); - - // Define the task pump - var pumpTask = Task.Run(async () => - { - // We process messages until cancellation is requested. - // When cancellation is requested, we continue processing messages until the memory queue is drained. - while (!cancellationToken.IsCancellationRequested || !queuedMessages.IsEmpty) - { - await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - - // Retrieved the next message from the queue and process it - var runningTask = Task.Run( - async () => - { - var messageProcessed = false; - - using (_metrics.Measure.Timer.Time(Metrics.MessageProcessingTimer)) - { - if (queuedMessages.TryDequeue(out (string TenantId, CloudMessage Message) result)) - { - var tenantInfo = GetTenantInfo(result.TenantId); - - try - { - // Process the message - OnMessage?.Invoke(result.TenantId, result.Message, cancellationToken); - - // Delete the processed message from the queue - // PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue even when the message pump is shutting down - await tenantInfo.QueueManager.DeleteMessageAsync(result.Message, CancellationToken.None).ConfigureAwait(false); - } - catch (Exception ex) - { - var isPoison = result.Message.DequeueCount > _maxDequeueCount; - OnError?.Invoke(result.TenantId, result.Message, ex, isPoison); - if (isPoison) - { - // PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue and moved to poison queue even when the message pump is shutting down - if (_poisonQueueManager != null) - { - result.Message.Metadata["PoisonExceptionMessage"] = ex.GetBaseException().Message; - result.Message.Metadata["PoisonExceptionDetails"] = ex.GetBaseException().ToString(); - - await _poisonQueueManager.AddMessageAsync(result.Message.Content, result.Message.Metadata, null, null, CancellationToken.None).ConfigureAwait(false); - } - - await tenantInfo.QueueManager.DeleteMessageAsync(result.Message, CancellationToken.None).ConfigureAwait(false); - } - } - - messageProcessed = true; - } - } - - // Increment the counter if we processed a message - if (messageProcessed) _metrics.Measure.Counter.Increment(Metrics.MessagesProcessedCounter); - - // Return a value indicating whether we processed a message or not - return messageProcessed; - }, - CancellationToken.None); - - // Add the task to the dictionary of tasks (allows us to keep track of the running tasks) - runningTasks.TryAdd(runningTask, runningTask); - - // Complete the task - runningTask.ContinueWith( - t => - { - semaphore.Release(); - runningTasks.TryRemove(t, out Task taskToBeRemoved); - }, - TaskContinuationOptions.ExecuteSynchronously) - .IgnoreAwait(); - } - }); - - // Run the task pump until canceled - await pumpTask.UntilCancelled().ConfigureAwait(false); - - // Task pump has been canceled, wait for the currently running tasks to complete - await Task.WhenAll(runningTasks.Values).UntilCancelled().ConfigureAwait(false); - } - - private async IAsyncEnumerable<(string TenantId, CloudMessage Message)> FetchMessages(TimeSpan? visibilityTimeout, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var messageCount = 0; - var originalTenant = _tenantIds.Current; - - using (_metrics.Measure.Timer.Time(Metrics.MessagesFetchingTimer)) - { - do - { - var tenantId = _tenantIds.MoveToNextItem(); - var tenantInfo = GetTenantInfo(tenantId); - - if (!cancellationToken.IsCancellationRequested && tenantInfo.LastFetched.Add(tenantInfo.FetchDelay) < DateTime.UtcNow) - { - IEnumerable messages = null; - - try - { - messages = await tenantInfo.QueueManager.GetMessagesAsync(_concurrentTasks, visibilityTimeout, cancellationToken).ConfigureAwait(false); - } - catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) - { - // The message pump is shutting down. - // This exception can be safely ignored. - } - catch (RequestFailedException rfe) when (rfe.ErrorCode == "QueueNotFound") - { - // The queue has been deleted - _tenantIds.Remove(tenantId); - _tenantQueueManagers.TryRemove(tenantId, out _); - } - catch (Exception e) - { - _logger?.LogError(e.GetBaseException(), "An error occured while fetching messages for tenant {tenantId}. The error was caught and ignored.", tenantId); - } - - if (messages != null && messages.Any()) - { - var messagesCount = messages.Count(); - _logger?.LogTrace("Fetched {messagesCount} message(s) for tenant {tenantId}.", messagesCount, tenantId); - - foreach (var message in messages) - { - Interlocked.Increment(ref messageCount); - yield return (tenantId, message); - } - - // Reset the Fetch delay to zero to indicate that we can fetch more messages from this queue as soon as possible - _tenantQueueManagers[tenantId] = new Lazy<(QueueManager, DateTime, TimeSpan)>(() => (tenantInfo.QueueManager, DateTime.UtcNow, TimeSpan.Zero)); - } - else - { - _logger?.LogTrace("There are no messages for tenant {tenantId}.", tenantId); - _metrics.Measure.Counter.Increment(Metrics.QueueEmptyCounter); - - // Set a "resonable" fetch delay to ensure we don't query an empty queue too often - var delay = tenantInfo.FetchDelay.Add(TimeSpan.FromSeconds(5)); - if (delay.TotalSeconds > 15) delay = TimeSpan.FromSeconds(15); - - _tenantQueueManagers[tenantId] = new Lazy<(QueueManager, DateTime, TimeSpan)>(() => (tenantInfo.QueueManager, DateTime.UtcNow, delay)); - } - } - } - - // Stop when we either retrieved the desired number of messages OR we have looped through all the known tenants - while (messageCount < (_concurrentTasks * 2) && (string.IsNullOrEmpty(originalTenant) || originalTenant != _tenantIds.Current)); - } - - if (messageCount == 0) - { - _logger?.LogTrace("All tenant queues are empty, no messages fetched."); - try - { - // All queues are empty - _metrics.Measure.Counter.Increment(Metrics.AllQueuesEmptyCounter); - OnEmpty?.Invoke(cancellationToken); - } - catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) - { - // The message pump is shutting down. - // This exception can be safely ignored. - } - catch (Exception e) - { - _logger?.LogError(e.GetBaseException(), "An error occured when handling empty queues. The error was caught and ignored."); - } - } - } - - private (QueueManager QueueManager, DateTime LastFetched, TimeSpan FetchDelay) GetTenantInfo(string tenantId) - { - var lazyQueueManager = _tenantQueueManagers.GetOrAdd(tenantId, tenantId => - { - return new Lazy<(QueueManager, DateTime, TimeSpan)>(() => - { - _tenantIds.Add(tenantId); - return (_queueManagerFactory.Invoke(tenantId), DateTime.MinValue, TimeSpan.Zero); - }); - }); - - return lazyQueueManager.Value; + await _messagePump.StartAsync(cancellationToken).ConfigureAwait(false); } #endregion diff --git a/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs index 41c38ac..087dd9c 100644 --- a/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs @@ -1,14 +1,8 @@ using App.Metrics; -using Azure.Storage.Blobs; -using Azure.Storage.Queues; -using Microsoft.Extensions.DependencyModel; using Microsoft.Extensions.Logging; -using Picton.Messaging.Messages; +using Picton.Messaging.Utilities; using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -22,11 +16,14 @@ public class AsyncMultiTenantMessagePumpWithHandlers { #region FIELDS - private static readonly ReaderWriterLockSlim _lock = new ReaderWriterLockSlim(); - private static IDictionary _messageHandlers; + private readonly MessagePumpOptions _messagePumpOptions; + private readonly string _queueNamePrefix; + private readonly TimeSpan? _visibilityTimeout; + private readonly int _maxDequeueCount; private readonly ILogger _logger; + private readonly AsyncMultiTenantMessagePump _messagePump; #endregion @@ -38,17 +35,26 @@ public class AsyncMultiTenantMessagePumpWithHandlers /// /// /// - /// OnError = (tenantId, message, exception, isPoison) => Trace.TraceError("An error occured: {0}", exception); + /// OnError = (message, exception, isPoison) => Trace.TraceError("An error occured: {0}", exception); /// /// /// /// When isPoison is set to true, you should copy this message to a poison queue because it will be deleted from the original queue. /// - public Action OnError - { - get { return _messagePump.OnError; } - set { _messagePump.OnError = value; } - } + public Action OnError { get; set; } + + /// + /// Gets or sets the logic to execute when all queues are empty. + /// + /// + /// + /// OnEmpty = cancellationToken => Task.Delay(2500, cancellationToken).Wait(); + /// + /// + /// + /// If this property is not set, the default logic is to do nothing. + /// + public Action OnEmpty { get; set; } #endregion @@ -57,62 +63,23 @@ public Action OnError /// /// Initializes a new instance of the class. /// - /// - /// A connection string includes the authentication information required for your application to access data in an Azure Storage account at runtime. - /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. - /// - /// Queues name prefix. - /// The number of concurrent tasks. - /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poinson' messages. + /// Options for the mesage pump. + /// The common prefix in the naming convention. /// The visibility timeout. /// The maximum dequeue count. - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request to the queue. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request to the blob storage. - /// /// The logger. /// The system where metrics are published. - [ExcludeFromCodeCoverage] - public AsyncMultiTenantMessagePumpWithHandlers(string connectionString, string queueNamePrefix, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, QueueClientOptions queueClientOptions = null, BlobClientOptions blobClientOptions = null, ILogger logger = null, IMetrics metrics = null) + public AsyncMultiTenantMessagePumpWithHandlers(MessagePumpOptions options, string queueNamePrefix, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) { - _logger = logger; + _messageHandlers = MessageHandlersDiscoverer.GetMessageHandlers(logger); - _messagePump = new AsyncMultiTenantMessagePump(connectionString, queueNamePrefix, concurrentTasks, poisonQueueName, visibilityTimeout, maxDequeueCount, queueClientOptions, blobClientOptions, logger, metrics) - { - OnMessage = (tenantId, message, cancellationToken) => - { - var contentType = message.Content.GetType(); - - if (!_messageHandlers.TryGetValue(contentType, out Type[] handlers)) - { - throw new Exception($"Received a message of type {contentType.FullName} but could not find a class implementing IMessageHandler<{contentType.FullName}>"); - } - - foreach (var handlerType in handlers) - { - object handler = null; - if (handlerType.GetConstructor(new[] { typeof(ILogger) }) != null) - { - handler = Activator.CreateInstance(handlerType, new[] { (object)logger }); - } - else - { - handler = Activator.CreateInstance(handlerType); - } - - var handlerMethod = handlerType.GetMethod("Handle", new[] { contentType }); - handlerMethod.Invoke(handler, new[] { message.Content }); - } - } - }; + _messagePumpOptions = options; + _queueNamePrefix = queueNamePrefix; + _visibilityTimeout = visibilityTimeout; + _maxDequeueCount = maxDequeueCount; + _logger = logger; - DiscoverMessageHandlersIfNecessary(logger); + _messagePump = new AsyncMultiTenantMessagePump(options, queueNamePrefix, visibilityTimeout, maxDequeueCount, logger, metrics); } #endregion @@ -127,111 +94,35 @@ public AsyncMultiTenantMessagePumpWithHandlers(string connectionString, string q /// A representing the asynchronous operation. public Task StartAsync(CancellationToken cancellationToken) { - return _messagePump.StartAsync(cancellationToken); - } - - #endregion - - #region PRIVATE METHODS - - private static void DiscoverMessageHandlersIfNecessary(ILogger logger) - { - try + _messagePump.OnEmpty = OnEmpty; + _messagePump.OnError = (queueName, message, exception, isPoison) => OnError?.Invoke(queueName.TrimStart(_queueNamePrefix), message, exception, isPoison); + _messagePump.OnMessage = (queueName, message, cancellationToken) => { - _lock.EnterUpgradeableReadLock(); + var contentType = message.Content.GetType(); - if (_messageHandlers == null) + if (!_messageHandlers.TryGetValue(contentType, out Type[] handlers)) { - try - { - _lock.EnterWriteLock(); + throw new Exception($"Received a message of type {contentType.FullName} but could not find a class implementing IMessageHandler<{contentType.FullName}>"); + } - _messageHandlers ??= GetMessageHandlers(null); + foreach (var handlerType in handlers) + { + object handler = null; + if (handlerType.GetConstructor(new[] { typeof(ILogger) }) != null) + { + handler = Activator.CreateInstance(handlerType, new[] { (object)_logger }); } - finally + else { - if (_lock.IsWriteLockHeld) _lock.ExitWriteLock(); + handler = Activator.CreateInstance(handlerType); } - } - } - finally - { - if (_lock.IsUpgradeableReadLockHeld) _lock.ExitUpgradeableReadLock(); - } - } - - private static IDictionary GetMessageHandlers(ILogger logger) - { - logger?.LogTrace("Discovering message handlers."); - var assemblies = GetLocalAssemblies(); - - var assembliesCount = assemblies.Length; - if (assembliesCount == 0) logger?.LogTrace($"Did not find any local assembly."); - else if (assembliesCount == 1) logger?.LogTrace("Found 1 local assembly."); - else logger?.LogTrace($"Found {assemblies.Length} local assemblies."); - - var typesWithMessageHandlerInterfaces = assemblies - .SelectMany(x => x.GetTypes()) - .Where(t => !t.GetTypeInfo().IsInterface) - .Select(type => new - { - Type = type, - MessageTypes = type - .GetInterfaces() - .Where(i => i.GetTypeInfo().IsGenericType) - .Where(i => i.GetGenericTypeDefinition() == typeof(IMessageHandler<>)) - .SelectMany(i => i.GetGenericArguments()) - }) - .Where(t => t.MessageTypes != null && t.MessageTypes.Any()) - .ToArray(); - - var classesCount = typesWithMessageHandlerInterfaces.Length; - if (classesCount == 0) logger?.LogTrace("Did not find any class implementing the 'IMessageHandler' interface."); - else if (classesCount == 1) logger?.LogTrace("Found 1 class implementing the 'IMessageHandler' interface."); - else logger?.LogTrace($"Found {typesWithMessageHandlerInterfaces.Length} classes implementing the 'IMessageHandler' interface."); - - var oneTypePerMessageHandler = typesWithMessageHandlerInterfaces - .SelectMany(t => t.MessageTypes, (t, messageType) => - new - { - t.Type, - MessageType = messageType - }) - .ToArray(); - - var messageHandlers = oneTypePerMessageHandler - .GroupBy(h => h.MessageType) - .ToDictionary(group => group.Key, group => group.Select(t => t.Type) - .ToArray()); - - return messageHandlers; - } - - private static Assembly[] GetLocalAssemblies() - { - var dependencies = DependencyContext.Default.RuntimeLibraries; - - var assemblies = new List(); - foreach (var library in dependencies) - { - if (IsCandidateLibrary(library)) - { - var assembly = Assembly.Load(new AssemblyName(library.Name)); - assemblies.Add(assembly); + var handlerMethod = handlerType.GetMethod("Handle", new[] { contentType }); + handlerMethod.Invoke(handler, new[] { message.Content }); } - } - - return assemblies.ToArray(); - } + }; - private static bool IsCandidateLibrary(RuntimeLibrary library) - { - return !library.Name.StartsWith("Microsoft.", StringComparison.OrdinalIgnoreCase) && - !library.Name.StartsWith("System.", StringComparison.OrdinalIgnoreCase) && - !library.Name.StartsWith("NetStandard.", StringComparison.OrdinalIgnoreCase) && - !string.Equals(library.Type, "package", StringComparison.OrdinalIgnoreCase) && - !string.Equals(library.Type, "referenceassembly", StringComparison.OrdinalIgnoreCase); + return _messagePump.StartAsync(cancellationToken); } #endregion diff --git a/Source/Picton.Messaging/Utilities/RoundRobinList.cs b/Source/Picton.Messaging/Utilities/RoundRobinList.cs index f13d588..c7963b2 100644 --- a/Source/Picton.Messaging/Utilities/RoundRobinList.cs +++ b/Source/Picton.Messaging/Utilities/RoundRobinList.cs @@ -20,13 +20,73 @@ public RoundRobinList(IEnumerable list) _linkedList = new LinkedList(list); } - public T Current => _current == default ? default : _current.Value; + public T Current + { + get + { + try + { + _lock.EnterReadLock(); + + return _current == default ? default : _current.Value; + } + finally + { + if (_lock.IsReadLockHeld) _lock.ExitReadLock(); + } + } + } + + public T Next + { + get + { + try + { + _lock.EnterReadLock(); + + return (_current == default || _current.Next == default) ? _linkedList.First.Value : _current.Next.Value; + } + finally + { + if (_lock.IsReadLockHeld) _lock.ExitReadLock(); + } + } + } + + public T Previous + { + get + { + try + { + _lock.EnterReadLock(); - public T Next => _current == default ? default : _current.Next.Value; + return (_current == default || _current.Previous == default) ? _linkedList.Last.Value : _current.Previous.Value; + } + finally + { + if (_lock.IsReadLockHeld) _lock.ExitReadLock(); + } + } + } - public T Previous => _current == default ? default : _current.Previous.Value; + public int Count + { + get + { + try + { + _lock.EnterReadLock(); - public int Count => _linkedList.Count; + return _linkedList.Count; + } + finally + { + if (_lock.IsReadLockHeld) _lock.ExitReadLock(); + } + } + } /// /// Reset the Round Robin to point to the first item. @@ -112,7 +172,7 @@ public T MoveToNextItem() /// Remove an item from the list. /// /// The item. - public bool Remove(T item) + public bool RemoveItem(T item) { try { @@ -126,7 +186,7 @@ public bool Remove(T item) } } - public void Add(T item) + public void AddItem(T item) { try { From 77b9ee7f902d807b69a1b8713469c2ee36455901 Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 7 Jan 2024 14:17:24 -0500 Subject: [PATCH 16/28] Add XML comments --- Source/Picton.Messaging/AsyncMessagePump.cs | 15 +++++++++++++++ .../AsyncMessagePumpWithHandlers.cs | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 60ed63c..88f1a09 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -94,11 +94,22 @@ public AsyncMessagePump(MessagePumpOptions options, ILogger logger = null, IMetr #region PUBLIC METHODS + /// + /// Add a queue to be monitored. + /// + /// The name of the queue. + /// Optional. The name of the queue where poison messages are automatically moved. + /// Optional. Specifies the visibility timeout value. The default value is 30 seconds. + /// Optional. A nonzero integer value that specifies the number of time we try to process a message before giving up and declaring the message to be "poison". The default value is 3. public void AddQueue(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3) { AddQueue(new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount)); } + /// + /// Add a queue to be monitored. + /// + /// Queue configuration. public void AddQueue(QueueConfig queueConfig) { if (string.IsNullOrEmpty(queueConfig.QueueName)) throw new ArgumentNullException(nameof(queueConfig.QueueName)); @@ -110,6 +121,10 @@ public void AddQueue(QueueConfig queueConfig) AddQueue(queueManager, poisonQueueManager, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); } + /// + /// Remove a queue from the list of queues that are monitored. + /// + /// The name of the queue. public void RemoveQueue(string queueName) { // Do not remove from _queuManagers because there could messages still in the memory queue that need to be processed diff --git a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs index cebfcf7..38030bc 100644 --- a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs @@ -74,16 +74,32 @@ public AsyncMessagePumpWithHandlers(MessagePumpOptions options, ILogger logger = #region PUBLIC METHODS + + /// + /// Add a queue to be monitored. + /// + /// The name of the queue. + /// Optional. The name of the queue where poison messages are automatically moved. + /// Optional. Specifies the visibility timeout value. The default value is 30 seconds. + /// Optional. A nonzero integer value that specifies the number of time we try to process a message before giving up and declaring the message to be "poison". The default value is 3. public void AddQueue(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3) { _messagePump.AddQueue(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount); } + /// + /// Add a queue to be monitored. + /// + /// Queue configuration. public void AddQueue(QueueConfig queueConfig) { _messagePump.AddQueue(queueConfig); } + /// + /// Remove a queue from the list of queues that are monitored. + /// + /// The name of the queue. public void RemoveQueue(string queueName) { _messagePump.RemoveQueue(queueName); From b4336d3d00a1565372680215724dee2a6f1b97a4 Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 7 Jan 2024 15:05:52 -0500 Subject: [PATCH 17/28] Configure message pump Resolves #34 --- .../TestsRunner.cs | 2 +- Source/Picton.Messaging/AsyncMessagePump.cs | 101 ++++++++++-------- .../AsyncMultiTenantMessagePump.cs | 14 +-- ...AsyncMultiTenantMessagePumpWithHandlers.cs | 5 +- Source/Picton.Messaging/MessagePumpOptions.cs | 60 ++++++++++- 5 files changed, 124 insertions(+), 58 deletions(-) diff --git a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs index 7079c42..d5ee7c2 100644 --- a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs +++ b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs @@ -208,7 +208,7 @@ private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, // Configure the message pump var cts = new CancellationTokenSource(); var options = new MessagePumpOptions(connectionString, concurrentTasks, null, null); - var messagePump = new AsyncMultiTenantMessagePump(options, queueNamePrefix, TimeSpan.FromMinutes(1), 3, _logger, metrics) + var messagePump = new AsyncMultiTenantMessagePump(options, queueNamePrefix, logger: _logger, metrics: metrics) { OnMessage = (tenantId, message, cancellationToken) => { diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 88f1a09..2ddd1fe 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -28,6 +28,7 @@ public class AsyncMessagePump private readonly MessagePumpOptions _messagePumpOptions; private readonly ILogger _logger; private readonly IMetrics _metrics; + private readonly bool _metricsTurnedOff; #endregion @@ -82,10 +83,12 @@ public AsyncMessagePump(MessagePumpOptions options, ILogger logger = null, IMetr if (options == null) throw new ArgumentNullException(nameof(options)); if (string.IsNullOrEmpty(options.ConnectionString)) throw new ArgumentNullException(nameof(options.ConnectionString)); if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); + if (options.FetchMessagesInterval <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(options.FetchMessagesInterval), "Fetch messages interval must be greather than zero"); _messagePumpOptions = options; _logger = logger; _metrics = metrics ?? TurnOffMetrics(); + _metricsTurnedOff = metrics == null; InitDefaultActions(); } @@ -200,66 +203,72 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) } } }, - TimeSpan.FromMilliseconds(500), + _messagePumpOptions.FetchMessagesInterval, cancellationToken, TaskCreationOptions.LongRunning); // Define the task that checks how many messages are queued in Azure - RecurrentCancellableTask.StartNew( - async () => - { - var count = 0; - foreach (var kvp in _queueManagers) + if (!_metricsTurnedOff && _messagePumpOptions.CountAzureMessagesInterval > TimeSpan.Zero) + { + RecurrentCancellableTask.StartNew( + async () => { - var queueName = kvp.Key; - (var queueConfig, var queueManager, var poisonQueueManager, var lastFetched, var fetchDelay) = kvp.Value; - - try + var count = 0; + foreach (var kvp in _queueManagers) { - var properties = await queueManager.GetPropertiesAsync(cancellationToken).ConfigureAwait(false); + var queueName = kvp.Key; + (var queueConfig, var queueManager, var poisonQueueManager, var lastFetched, var fetchDelay) = kvp.Value; - count += properties.ApproximateMessagesCount; - } - catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) - { - // The message pump is shutting down. - // This exception can be safely ignored. + try + { + var properties = await queueManager.GetPropertiesAsync(cancellationToken).ConfigureAwait(false); + + count += properties.ApproximateMessagesCount; + } + catch (Exception e) when (e is TaskCanceledException || e is OperationCanceledException) + { + // The message pump is shutting down. + // This exception can be safely ignored. + } + catch (RequestFailedException rfe) when (rfe.ErrorCode == "QueueNotFound") + { + // The queue has been deleted + RemoveQueue(queueName); + } + catch (Exception e) + { + _logger?.LogError(e.GetBaseException(), "An error occured while checking how many message are waiting in Azure. The error was caught and ignored."); + } } - catch (RequestFailedException rfe) when (rfe.ErrorCode == "QueueNotFound") + + _metrics.Measure.Gauge.SetValue(Metrics.QueuedCloudMessagesGauge, count); + }, + _messagePumpOptions.CountAzureMessagesInterval, + cancellationToken, + TaskCreationOptions.LongRunning); + } + + // Define the task that checks how many messages are queued in memory + if (!_metricsTurnedOff && _messagePumpOptions.CountMemoryMessagesInterval > TimeSpan.Zero) + { + RecurrentCancellableTask.StartNew( + () => + { + try { - // The queue has been deleted - RemoveQueue(queueName); + _metrics.Measure.Gauge.SetValue(Metrics.QueuedMemoryMessagesGauge, queuedMessages.Count); } catch (Exception e) { - _logger?.LogError(e.GetBaseException(), "An error occured while checking how many message are waiting in Azure. The error was caught and ignored."); + _logger?.LogError(e.GetBaseException(), "An error occured while checking how many message are waiting in the memory queue. The error was caught and ignored."); } - } - - _metrics.Measure.Gauge.SetValue(Metrics.QueuedCloudMessagesGauge, count); - }, - TimeSpan.FromMilliseconds(5000), - cancellationToken, - TaskCreationOptions.LongRunning); - // Define the task that checks how many messages are queued in memory - RecurrentCancellableTask.StartNew( - () => - { - try - { - _metrics.Measure.Gauge.SetValue(Metrics.QueuedMemoryMessagesGauge, queuedMessages.Count); - } - catch (Exception e) - { - _logger?.LogError(e.GetBaseException(), "An error occured while checking how many message are waiting in the memory queue. The error was caught and ignored."); - } - - return Task.CompletedTask; - }, - TimeSpan.FromMilliseconds(5000), - cancellationToken, - TaskCreationOptions.LongRunning); + return Task.CompletedTask; + }, + TimeSpan.FromMilliseconds(5000), + cancellationToken, + TaskCreationOptions.LongRunning); + } // Define the task pump var pumpTask = Task.Run(async () => diff --git a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs index 4d5103f..acd100b 100644 --- a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs @@ -29,8 +29,11 @@ public class AsyncMultiTenantMessagePump { #region FIELDS + private static readonly TimeSpan _defaultDiscoverQueuesInterval = TimeSpan.FromSeconds(30); + private readonly MessagePumpOptions _messagePumpOptions; private readonly string _queueNamePrefix; + private readonly TimeSpan _discoverQueuesInterval; private readonly TimeSpan? _visibilityTimeout; private readonly int _maxDequeueCount; private readonly ILogger _logger; @@ -83,19 +86,18 @@ public class AsyncMultiTenantMessagePump /// /// Options for the mesage pump. /// The common prefix in the naming convention. + /// The frequency we check for queues in the Azure storage account matching the naming convention. Default is 30 seconds. /// The visibility timeout. /// The maximum dequeue count. /// The logger. /// The system where metrics are published. - public AsyncMultiTenantMessagePump(MessagePumpOptions options, string queueNamePrefix, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + public AsyncMultiTenantMessagePump(MessagePumpOptions options, string queueNamePrefix, TimeSpan? discoverQueuesInterval = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) { - if (options == null) throw new ArgumentNullException(nameof(options)); - if (string.IsNullOrEmpty(options.ConnectionString)) throw new ArgumentNullException(nameof(options.ConnectionString)); - if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); - if (string.IsNullOrEmpty(queueNamePrefix)) throw new ArgumentNullException(nameof(queueNamePrefix)); + if (discoverQueuesInterval != null && discoverQueuesInterval <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(discoverQueuesInterval), "The 'discover queues' interval must be greater than zero."); _messagePumpOptions = options; _queueNamePrefix = queueNamePrefix; + _discoverQueuesInterval = discoverQueuesInterval ?? _defaultDiscoverQueuesInterval; _visibilityTimeout = visibilityTimeout; _maxDequeueCount = maxDequeueCount; _logger = logger; @@ -156,7 +158,7 @@ public async Task StartAsync(CancellationToken cancellationToken) _logger?.LogError(e.GetBaseException(), "An error occured while fetching the Azure queues that match the naming convention. The error was caught and ignored."); } }, - TimeSpan.FromMilliseconds(30000), + _discoverQueuesInterval, cancellationToken, TaskCreationOptions.LongRunning); diff --git a/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs index 087dd9c..b941e29 100644 --- a/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs @@ -65,11 +65,12 @@ public class AsyncMultiTenantMessagePumpWithHandlers /// /// Options for the mesage pump. /// The common prefix in the naming convention. + /// The frequency we check for queues in the Azure storage account matching the naming convention. Default is 30 seconds. /// The visibility timeout. /// The maximum dequeue count. /// The logger. /// The system where metrics are published. - public AsyncMultiTenantMessagePumpWithHandlers(MessagePumpOptions options, string queueNamePrefix, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) + public AsyncMultiTenantMessagePumpWithHandlers(MessagePumpOptions options, string queueNamePrefix, TimeSpan? discoverQueuesInterval = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, ILogger logger = null, IMetrics metrics = null) { _messageHandlers = MessageHandlersDiscoverer.GetMessageHandlers(logger); @@ -79,7 +80,7 @@ public AsyncMultiTenantMessagePumpWithHandlers(MessagePumpOptions options, strin _maxDequeueCount = maxDequeueCount; _logger = logger; - _messagePump = new AsyncMultiTenantMessagePump(options, queueNamePrefix, visibilityTimeout, maxDequeueCount, logger, metrics); + _messagePump = new AsyncMultiTenantMessagePump(options, queueNamePrefix, discoverQueuesInterval, visibilityTimeout, maxDequeueCount, logger, metrics); } #endregion diff --git a/Source/Picton.Messaging/MessagePumpOptions.cs b/Source/Picton.Messaging/MessagePumpOptions.cs index e003e52..2db7db7 100644 --- a/Source/Picton.Messaging/MessagePumpOptions.cs +++ b/Source/Picton.Messaging/MessagePumpOptions.cs @@ -4,14 +4,41 @@ namespace Picton.Messaging { + /// + /// Configuration options for a message pump. + /// public record MessagePumpOptions { - public MessagePumpOptions(string connectionString, int concurrentTasks, QueueClientOptions queueClientOptions = null, BlobClientOptions blobClientOptions = null) + private const int _defaultConcurrentTasks = 25; + private static readonly TimeSpan _defaultFetchMessagesInterval = TimeSpan.FromSeconds(1); + private static readonly TimeSpan _defaultCountAzureMessagesInterval = TimeSpan.FromSeconds(5); + private static readonly TimeSpan _defaultCountMemoryMessagesInterval = TimeSpan.FromSeconds(5); + + /// + /// Initializes a new instance of the class. + /// + public MessagePumpOptions() + { } + + /// + /// Initializes a new instance of the class. + /// + /// The connection string. + /// The number of concurrent tasks. In other words: the number of messages that can be processed at a time. + /// The client options that define the transport pipeline policies for authentication, retries, etc., that are applied to every request to the queue. + /// The client options that define the transport pipeline policies for authentication, retries, etc., that are applied to every request to the blob storage. + /// The frequency at which messages are fetched from the Azure queues. The default value is 1 second. + /// The frequency at which we count how many messages are queue in Azure, waiting to be fetched. Default is 5 seconds. + /// the frequency at which we count how many messages have been fetched from Azure and are queued in memory, waiting to be processed. Default is 5 seconds. + public MessagePumpOptions(string connectionString, int? concurrentTasks, QueueClientOptions queueClientOptions = null, BlobClientOptions blobClientOptions = null, TimeSpan? fetchMessagesInterval = null, TimeSpan? countAzureMessagesInterval = null, TimeSpan? countMemoryMessagesInterval = null) { ConnectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString)); - ConcurrentTasks = concurrentTasks; + ConcurrentTasks = concurrentTasks ?? _defaultConcurrentTasks; QueueClientOptions = queueClientOptions; BlobClientOptions = blobClientOptions; + FetchMessagesInterval = fetchMessagesInterval ?? _defaultFetchMessagesInterval; + CountAzureMessagesInterval = countAzureMessagesInterval ?? _defaultCountAzureMessagesInterval; + CountMemoryMessagesInterval = countMemoryMessagesInterval ?? _defaultCountMemoryMessagesInterval; } /// @@ -23,7 +50,7 @@ public MessagePumpOptions(string connectionString, int concurrentTasks, QueueCli /// /// Gets or sets the number of concurrent tasks. In other words: the number of messages that can be processed at a time. /// - public int ConcurrentTasks { get; set; } = 25; + public int ConcurrentTasks { get; set; } = _defaultConcurrentTasks; /// /// Gets or sets the optional client options that define the transport @@ -38,5 +65,32 @@ public MessagePumpOptions(string connectionString, int concurrentTasks, QueueCli /// to every request to the blob storage. /// public BlobClientOptions BlobClientOptions { get; set; } = null; + + /// + /// Gets or sets the frequency at which messages are fetched from the Azure queues. The default value is 1 second. + /// + public TimeSpan FetchMessagesInterval { get; set; } = _defaultFetchMessagesInterval; + + /// + /// Gets or sets the frequency at which we count how many messages are queue in Azure, waiting to be processed. + /// The count is subsequently published to the metric system you have configured. + /// + /// You can turn off this behavior by setting this interval to `TimeSpan.Zero`. + /// + /// Default value is 5 seconds. + /// + /// This setting is ignored if you don't specify the sytem where metrics are published. + public TimeSpan CountAzureMessagesInterval { get; set; } = _defaultCountAzureMessagesInterval; + + /// + /// Gets or sets the frequency at which we count how many messages have been fetched from Azure and are queued in memory, waiting to be fetched. + /// The count is subsequently published to the metric system you have configured. + /// + /// You can turn off this behavior by setting this interval to `TimeSpan.Zero`. + /// + /// Default value is 5 seconds. + /// + /// This setting is ignored if you don't specify the sytem where metrics are published. + public TimeSpan CountMemoryMessagesInterval { get; set; } = _defaultCountMemoryMessagesInterval; } } From 9039f9b73273642229f798ca619200b305168c1c Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 7 Jan 2024 15:21:07 -0500 Subject: [PATCH 18/28] Upgrade xunit nuget package to 2.6.5 Also, install the xUnit analyzers package --- .../Picton.Messaging.UnitTests.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj b/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj index 7936cc2..5bf32ec 100644 --- a/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj +++ b/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj @@ -18,7 +18,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + + all runtime; build; native; contentfiles; analyzers From e4a13ebda017f44da9847c60259b35c481a8fa58 Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 7 Jan 2024 15:22:33 -0500 Subject: [PATCH 19/28] Implement changes suggested by analyzers --- .../TestsRunner.cs | 51 +++++++++---------- Source/Picton.Messaging/AsyncMessagePump.cs | 10 ++-- .../AsyncMessagePumpWithHandlers.cs | 9 ++-- ...AsyncMultiTenantMessagePumpWithHandlers.cs | 14 ++--- Source/Picton.Messaging/Extensions.cs | 2 - Source/Picton.Messaging/Metrics.cs | 14 ++--- .../Utilities/MessageHandlersDiscoverer.cs | 2 +- .../Utilities/RoundRobinList.cs | 23 ++++----- 8 files changed, 56 insertions(+), 69 deletions(-) diff --git a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs index d5ee7c2..03c4f6a 100644 --- a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs +++ b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs @@ -10,7 +10,7 @@ namespace Picton.Messaging.IntegrationTests { - internal class TestsRunner + internal class TestsRunner(ILogger logger) { private enum ResultCodes { @@ -19,12 +19,7 @@ private enum ResultCodes Cancelled = 1223 } - private readonly ILogger _logger; - - public TestsRunner(ILogger logger) - { - _logger = logger; - } + private readonly ILogger _logger = logger; public async Task RunAsync() { @@ -102,10 +97,10 @@ private async Task RunAsyncMessagePumpTests(string connectionString, string queu // Add messages to the queue _logger.LogInformation("Adding {numberOfMessages} string messages to the {queueName} queue...", numberOfMessages, queueName); var queueManager = new QueueManager(connectionString, queueName); - await queueManager.ClearAsync().ConfigureAwait(false); + await queueManager.ClearAsync(cancellationToken).ConfigureAwait(false); for (var i = 0; i < numberOfMessages; i++) { - await queueManager.AddMessageAsync($"Hello world {i}").ConfigureAwait(false); + await queueManager.AddMessageAsync($"Hello world {i}", cancellationToken: cancellationToken).ConfigureAwait(false); } // Configure the message pump @@ -115,7 +110,7 @@ private async Task RunAsyncMessagePumpTests(string connectionString, string queu { OnMessage = (queueName, message, cancellationToken) => { - _logger.LogInformation(message.Content.ToString()); + _logger.LogInformation("{messageContent}", message.Content.ToString()); } }; messagePump.AddQueue(queueName, null, TimeSpan.FromMinutes(1), 3); @@ -138,7 +133,7 @@ private async Task RunAsyncMessagePumpTests(string connectionString, string queu await messagePump.StartAsync(cts.Token).ConfigureAwait(false); // Display summary - _logger.LogInformation($"\tDone in {sw.Elapsed.ToDurationString()}"); + _logger.LogInformation("\tDone in {duration}", sw.Elapsed.ToDurationString()); } private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, string queueName, int concurrentTasks, int numberOfMessages, IMetrics metrics, CancellationToken cancellationToken) @@ -151,10 +146,10 @@ private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, // Add messages to the queue _logger.LogInformation("Adding {numberOfMessages} messages with handlers to the {queueName} queue...", numberOfMessages, queueName); var queueManager = new QueueManager(connectionString, queueName); - await queueManager.ClearAsync().ConfigureAwait(false); + await queueManager.ClearAsync(cancellationToken).ConfigureAwait(false); for (var i = 0; i < numberOfMessages; i++) { - await queueManager.AddMessageAsync(new MyMessage { MessageContent = $"Hello world {i}" }).ConfigureAwait(false); + await queueManager.AddMessageAsync(new MyMessage { MessageContent = $"Hello world {i}" }, cancellationToken: cancellationToken).ConfigureAwait(false); } // Configure the message pump @@ -181,7 +176,7 @@ private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, await messagePump.StartAsync(cts.Token); // Display summary - _logger.LogInformation($"\tDone in {sw.Elapsed.ToDurationString()}"); + _logger.LogInformation("\tDone in {duration}", sw.Elapsed.ToDurationString()); } private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, string queueNamePrefix, int concurrentTasks, int[] numberOfMessagesForTenant, IMetrics metrics, CancellationToken cancellationToken) @@ -195,10 +190,10 @@ private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, for (int i = 0; i < numberOfMessagesForTenant.Length; i++) { var queueManager = new QueueManager(connectionString, $"{queueNamePrefix}{i:00}"); - await queueManager.ClearAsync().ConfigureAwait(false); + await queueManager.ClearAsync(cancellationToken).ConfigureAwait(false); for (var j = 0; j < numberOfMessagesForTenant[i]; j++) { - await queueManager.AddMessageAsync($"Hello world {j:00} to tenant {i:00}").ConfigureAwait(false); + await queueManager.AddMessageAsync($"Hello world {j:00} to tenant {i:00}", cancellationToken: cancellationToken).ConfigureAwait(false); } } @@ -213,19 +208,19 @@ private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, OnMessage = (tenantId, message, cancellationToken) => { var messageContent = message.Content.ToString(); - _logger.LogInformation($"{tenantId} - {messageContent}", tenantId, messageContent); - } - }; + _logger.LogInformation("{tenantId} - {messageContent}", tenantId, messageContent); + }, - // Stop the message pump when all tenant queues are empty. - messagePump.OnEmpty = cancellationToken => - { - // Stop the timer - if (sw.IsRunning) sw.Stop(); + // Stop the timer and the message pump when all tenant queues are empty. + OnEmpty = cancellationToken => + { + // Stop the timer + if (sw.IsRunning) sw.Stop(); - // Stop the message pump - _logger.LogDebug("Asking the multi-tenant message pump to stop..."); - cts.Cancel(); + // Stop the message pump + _logger.LogDebug("Asking the multi-tenant message pump to stop..."); + cts.Cancel(); + } }; // Start the message pump @@ -234,7 +229,7 @@ private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, await messagePump.StartAsync(cts.Token); // Display summary - _logger.LogInformation($"\tDone in {sw.Elapsed.ToDurationString()}"); + _logger.LogInformation("\tDone in {duration}", sw.Elapsed.ToDurationString()); } } } diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 2ddd1fe..6c9136d 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -22,8 +22,8 @@ public class AsyncMessagePump { #region FIELDS - private readonly ConcurrentDictionary _queueManagers = new ConcurrentDictionary(); - private readonly RoundRobinList _queueNames = new RoundRobinList(Enumerable.Empty()); + private readonly ConcurrentDictionary _queueManagers = new(); + private readonly RoundRobinList _queueNames = new(Enumerable.Empty()); private readonly MessagePumpOptions _messagePumpOptions; private readonly ILogger _logger; @@ -130,8 +130,10 @@ public void AddQueue(QueueConfig queueConfig) /// The name of the queue. public void RemoveQueue(string queueName) { - // Do not remove from _queuManagers because there could messages still in the memory queue that need to be processed - //_queueManagers.TryRemove(queueName, out _); + /* + * Do not remove from _queuManagers because there could messages still in the memory queue that need to be processed + * _queueManagers.TryRemove(queueName, out _); + */ _queueNames.RemoveItem(queueName); } diff --git a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs index 38030bc..e1e0a19 100644 --- a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs @@ -74,7 +74,6 @@ public AsyncMessagePumpWithHandlers(MessagePumpOptions options, ILogger logger = #region PUBLIC METHODS - /// /// Add a queue to be monitored. /// @@ -127,17 +126,17 @@ public Task StartAsync(CancellationToken cancellationToken) foreach (var handlerType in handlers) { object handler = null; - if (handlerType.GetConstructor(new[] { typeof(ILogger) }) != null) + if (handlerType.GetConstructor([typeof(ILogger)]) != null) { - handler = Activator.CreateInstance(handlerType, new[] { (object)_logger }); + handler = Activator.CreateInstance(handlerType, [(object)_logger]); } else { handler = Activator.CreateInstance(handlerType); } - var handlerMethod = handlerType.GetMethod("Handle", new[] { contentType }); - handlerMethod.Invoke(handler, new[] { message.Content }); + var handlerMethod = handlerType.GetMethod("Handle", [contentType]); + handlerMethod.Invoke(handler, [message.Content]); } }; diff --git a/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs index b941e29..c32c727 100644 --- a/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMultiTenantMessagePumpWithHandlers.cs @@ -18,10 +18,7 @@ public class AsyncMultiTenantMessagePumpWithHandlers private static IDictionary _messageHandlers; - private readonly MessagePumpOptions _messagePumpOptions; private readonly string _queueNamePrefix; - private readonly TimeSpan? _visibilityTimeout; - private readonly int _maxDequeueCount; private readonly ILogger _logger; private readonly AsyncMultiTenantMessagePump _messagePump; @@ -74,10 +71,7 @@ public AsyncMultiTenantMessagePumpWithHandlers(MessagePumpOptions options, strin { _messageHandlers = MessageHandlersDiscoverer.GetMessageHandlers(logger); - _messagePumpOptions = options; _queueNamePrefix = queueNamePrefix; - _visibilityTimeout = visibilityTimeout; - _maxDequeueCount = maxDequeueCount; _logger = logger; _messagePump = new AsyncMultiTenantMessagePump(options, queueNamePrefix, discoverQueuesInterval, visibilityTimeout, maxDequeueCount, logger, metrics); @@ -109,17 +103,17 @@ public Task StartAsync(CancellationToken cancellationToken) foreach (var handlerType in handlers) { object handler = null; - if (handlerType.GetConstructor(new[] { typeof(ILogger) }) != null) + if (handlerType.GetConstructor([typeof(ILogger)]) != null) { - handler = Activator.CreateInstance(handlerType, new[] { (object)_logger }); + handler = Activator.CreateInstance(handlerType, [(object)_logger]); } else { handler = Activator.CreateInstance(handlerType); } - var handlerMethod = handlerType.GetMethod("Handle", new[] { contentType }); - handlerMethod.Invoke(handler, new[] { message.Content }); + var handlerMethod = handlerType.GetMethod("Handle", [contentType]); + handlerMethod.Invoke(handler, [message.Content]); } }; diff --git a/Source/Picton.Messaging/Extensions.cs b/Source/Picton.Messaging/Extensions.cs index cba4d8b..20d99c8 100644 --- a/Source/Picton.Messaging/Extensions.cs +++ b/Source/Picton.Messaging/Extensions.cs @@ -15,11 +15,9 @@ internal static class Extensions /// The purpose of this extension method is to avoid a Visual Studio warning about async calls that are not awaited. /// /// The task. -#pragma warning disable RECS0154 // Parameter is never used #pragma warning disable IDE0060 // Remove unused parameter public static void IgnoreAwait(this Task task) #pragma warning restore IDE0060 // Remove unused parameter -#pragma warning restore RECS0154 // Parameter is never used { // Intentionaly left blank. } diff --git a/Source/Picton.Messaging/Metrics.cs b/Source/Picton.Messaging/Metrics.cs index 5c6dab6..4a782ec 100644 --- a/Source/Picton.Messaging/Metrics.cs +++ b/Source/Picton.Messaging/Metrics.cs @@ -10,7 +10,7 @@ internal static class Metrics /// /// Gets the counter indicating the number of messages processed by the message pump. /// - public static CounterOptions MessagesProcessedCounter => new CounterOptions + public static CounterOptions MessagesProcessedCounter => new() { Context = "Picton", Name = "MessagesProcessedCount", @@ -20,7 +20,7 @@ internal static class Metrics /// /// Gets the timer indicating the time it takes to process a message. /// - public static TimerOptions MessageProcessingTimer => new TimerOptions + public static TimerOptions MessageProcessingTimer => new() { Context = "Picton", Name = "MessageProcessingTime" @@ -29,7 +29,7 @@ internal static class Metrics /// /// Gets the timer indicating the time it takes to fetch a batch of messages from the Azure queue. /// - public static TimerOptions MessagesFetchingTimer => new TimerOptions + public static TimerOptions MessagesFetchingTimer => new() { Context = "Picton", Name = "MessagesFetchingTime" @@ -38,7 +38,7 @@ internal static class Metrics /// /// Gets the counter indicating the number of times we attempted to fetch messages from an Azure queue but it was empty. /// - public static CounterOptions QueueEmptyCounter => new CounterOptions + public static CounterOptions QueueEmptyCounter => new() { Context = "Picton", Name = "QueueEmptyCount" @@ -47,7 +47,7 @@ internal static class Metrics /// /// Gets the counter indicating the number of times we attempted to fetch messages from Azure but all the queues are empty. /// - public static CounterOptions AllQueuesEmptyCounter => new CounterOptions + public static CounterOptions AllQueuesEmptyCounter => new() { Context = "Picton", Name = "AllQueuesEmptyCount" @@ -56,7 +56,7 @@ internal static class Metrics /// /// Gets the gauge indicating the number of messages waiting in the Azure queue over time. /// - public static GaugeOptions QueuedCloudMessagesGauge => new GaugeOptions + public static GaugeOptions QueuedCloudMessagesGauge => new() { Context = "Picton", Name = "QueuedCloudMessages", @@ -66,7 +66,7 @@ internal static class Metrics /// /// Gets the gauge indicating the number of messages waiting in the memory queue over time. /// - public static GaugeOptions QueuedMemoryMessagesGauge => new GaugeOptions + public static GaugeOptions QueuedMemoryMessagesGauge => new() { Context = "Picton", Name = "QueuedMemoryMessages", diff --git a/Source/Picton.Messaging/Utilities/MessageHandlersDiscoverer.cs b/Source/Picton.Messaging/Utilities/MessageHandlersDiscoverer.cs index c82664d..6c30b0c 100644 --- a/Source/Picton.Messaging/Utilities/MessageHandlersDiscoverer.cs +++ b/Source/Picton.Messaging/Utilities/MessageHandlersDiscoverer.cs @@ -72,7 +72,7 @@ private static Assembly[] GetLocalAssemblies() } } - return assemblies.ToArray(); + return [.. assemblies]; } private static bool IsCandidateLibrary(RuntimeLibrary library) diff --git a/Source/Picton.Messaging/Utilities/RoundRobinList.cs b/Source/Picton.Messaging/Utilities/RoundRobinList.cs index c7963b2..8b0dedc 100644 --- a/Source/Picton.Messaging/Utilities/RoundRobinList.cs +++ b/Source/Picton.Messaging/Utilities/RoundRobinList.cs @@ -5,21 +5,16 @@ namespace Picton.Messaging.Utilities { - internal class RoundRobinList + /// + /// Initializes a new instance of the class. + /// + /// The items. + internal class RoundRobinList(IEnumerable list) { - private static readonly ReaderWriterLockSlim _lock = new ReaderWriterLockSlim(); - private readonly LinkedList _linkedList; + private static readonly ReaderWriterLockSlim _lock = new(); + private readonly LinkedList _linkedList = new(list); private LinkedListNode _current; - /// - /// Initializes a new instance of the class. - /// - /// The items. - public RoundRobinList(IEnumerable list) - { - _linkedList = new LinkedList(list); - } - public T Current { get @@ -186,6 +181,10 @@ public bool RemoveItem(T item) } } + /// + /// Add an item to the list. + /// + /// The item. public void AddItem(T item) { try From c3bf3d37536989beafb1ea413dc20f8e31e4e36c Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 28 Jan 2024 13:50:26 -0500 Subject: [PATCH 20/28] Refresh resource files --- build.cake | 2 +- global.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.cake b/build.cake index 12e8934..d021908 100644 --- a/build.cake +++ b/build.cake @@ -3,7 +3,7 @@ #tool dotnet:?package=coveralls.net&version=4.0.1 #tool nuget:https://f.feedz.io/jericho/jericho/nuget/?package=GitReleaseManager&version=0.17.0-collaborators0003 #tool nuget:?package=ReportGenerator&version=5.2.0 -#tool nuget:?package=xunit.runner.console&version=2.6.4 +#tool nuget:?package=xunit.runner.console&version=2.6.6 #tool nuget:?package=CodecovUploader&version=0.7.1 // Install addins. diff --git a/global.json b/global.json index 8e621ca..f43378f 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.100", + "version": "8.0.101", "rollForward": "patch", "allowPrerelease": false } From 7c0ec3ec7bd0d666e2a37afc0f814de30c741c3f Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 28 Jan 2024 14:31:55 -0500 Subject: [PATCH 21/28] (GH-35) Replace the internal ConcurrentQueue with a 'Channel' --- Source/Picton.Messaging/AsyncMessagePump.cs | 23 ++++++++++++++----- .../Picton.Messaging/Picton.Messaging.csproj | 2 ++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 6c9136d..7a85915 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; namespace Picton.Messaging @@ -190,20 +191,30 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) { var runningTasks = new ConcurrentDictionary(); var semaphore = new SemaphoreSlim(_messagePumpOptions.ConcurrentTasks, _messagePumpOptions.ConcurrentTasks); - var queuedMessages = new ConcurrentQueue<(string QueueName, CloudMessage Message)>(); + var channelOptions = new UnboundedChannelOptions() { SingleReader = false, SingleWriter = true }; + var channel = Channel.CreateUnbounded<(string QueueName, CloudMessage Message)>(channelOptions); + var channelCompleted = false; // Define the task that fetches messages from the Azure queue RecurrentCancellableTask.StartNew( async () => { // Fetch messages from Azure when the number of items in the concurrent queue falls below an "acceptable" level. - if (!cancellationToken.IsCancellationRequested && queuedMessages.Count <= _messagePumpOptions.ConcurrentTasks / 2) + if (!cancellationToken.IsCancellationRequested && + !channelCompleted && + channel.Reader.Count <= _messagePumpOptions.ConcurrentTasks / 2) { await foreach (var message in FetchMessages(cancellationToken)) { - queuedMessages.Enqueue(message); + await channel.Writer.WriteAsync(message).ConfigureAwait(false); } } + + // Mark the channel as "complete" which means that no more messages will be written to it + else if (!channelCompleted) + { + channelCompleted = channel.Writer.TryComplete(); + } }, _messagePumpOptions.FetchMessagesInterval, cancellationToken, @@ -258,7 +269,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) { try { - _metrics.Measure.Gauge.SetValue(Metrics.QueuedMemoryMessagesGauge, queuedMessages.Count); + _metrics.Measure.Gauge.SetValue(Metrics.QueuedMemoryMessagesGauge, channel.Reader.Count); } catch (Exception e) { @@ -277,7 +288,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) { // We process messages until cancellation is requested. // When cancellation is requested, we continue processing messages until the memory queue is drained. - while (!cancellationToken.IsCancellationRequested || !queuedMessages.IsEmpty) + while (!cancellationToken.IsCancellationRequested || channel.Reader.Count > 0) { await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -287,7 +298,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) { var messageProcessed = false; - if (queuedMessages.TryDequeue(out (string QueueName, CloudMessage Message) result)) + if (channel.Reader.TryRead(out (string QueueName, CloudMessage Message) result)) { if (_queueManagers.TryGetValue(result.QueueName, out (QueueConfig Config, QueueManager QueueManager, QueueManager PoisonQueueManager, DateTime LastFetched, TimeSpan FetchDelay) queueInfo)) { diff --git a/Source/Picton.Messaging/Picton.Messaging.csproj b/Source/Picton.Messaging/Picton.Messaging.csproj index b3b89ee..b0a7d59 100644 --- a/Source/Picton.Messaging/Picton.Messaging.csproj +++ b/Source/Picton.Messaging/Picton.Messaging.csproj @@ -47,10 +47,12 @@ + + From 956a68f673e855b87a47f1b1745c61850d8ed09d Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 28 Jan 2024 14:33:20 -0500 Subject: [PATCH 22/28] Minor fixes --- Source/Picton.Messaging/AsyncMessagePump.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 7a85915..6b10272 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -204,7 +204,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) !channelCompleted && channel.Reader.Count <= _messagePumpOptions.ConcurrentTasks / 2) { - await foreach (var message in FetchMessages(cancellationToken)) + await foreach (var message in FetchMessages(cancellationToken).ConfigureAwait(false)) { await channel.Writer.WriteAsync(message).ConfigureAwait(false); } @@ -367,7 +367,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) t => { semaphore.Release(); - runningTasks.TryRemove(t, out Task taskToBeRemoved); + runningTasks.TryRemove(t, out Task _); }, TaskContinuationOptions.ExecuteSynchronously) .IgnoreAwait(); @@ -387,7 +387,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) if (_queueNames.Count == 0) { - _logger?.LogTrace("There are no tenant queues being monitored. Therefore no messages could be fetched."); + _logger?.LogTrace("There are no queues being monitored. Therefore no messages could be fetched."); yield break; } @@ -444,7 +444,7 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) _logger?.LogTrace("There are no messages in {queueName}.", queueName); _metrics.Measure.Counter.Increment(Metrics.QueueEmptyCounter); - // Set a "resonable" fetch delay to ensure we don't query an empty queue too often + // Set a "reasonable" fetch delay to ensure we don't query an empty queue too often var delay = queueInfo.FetchDelay.Add(TimeSpan.FromSeconds(5)); if (delay.TotalSeconds > 15) delay = TimeSpan.FromSeconds(15); From 2edd264f4e901e2c0b0e36fb21c0cedfe5311dd1 Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 28 Jan 2024 15:46:09 -0500 Subject: [PATCH 23/28] the fetch delay when a queue is found to be empty should be configurable --- Source/Picton.Messaging/AsyncMessagePump.cs | 6 +++-- Source/Picton.Messaging/MessagePumpOptions.cs | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 6b10272..a16ec99 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -85,6 +85,8 @@ public AsyncMessagePump(MessagePumpOptions options, ILogger logger = null, IMetr if (string.IsNullOrEmpty(options.ConnectionString)) throw new ArgumentNullException(nameof(options.ConnectionString)); if (options.ConcurrentTasks < 1) throw new ArgumentOutOfRangeException(nameof(options.ConcurrentTasks), "Number of concurrent tasks must be greather than zero"); if (options.FetchMessagesInterval <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(options.FetchMessagesInterval), "Fetch messages interval must be greather than zero"); + if (options.EmptyQueueFetchDelay <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(options.EmptyQueueFetchDelay), "Emnpty queue fetch delay must be greather than zero"); + if (options.EmptyQueueMaxFetchDelay < options.EmptyQueueFetchDelay) throw new ArgumentOutOfRangeException(nameof(options.EmptyQueueMaxFetchDelay), "Max fetch delay can not be smaller than fetch delay"); _messagePumpOptions = options; _logger = logger; @@ -445,8 +447,8 @@ private async Task ProcessMessagesAsync(CancellationToken cancellationToken) _metrics.Measure.Counter.Increment(Metrics.QueueEmptyCounter); // Set a "reasonable" fetch delay to ensure we don't query an empty queue too often - var delay = queueInfo.FetchDelay.Add(TimeSpan.FromSeconds(5)); - if (delay.TotalSeconds > 15) delay = TimeSpan.FromSeconds(15); + var delay = queueInfo.FetchDelay.Add(_messagePumpOptions.EmptyQueueFetchDelay); + if (delay > _messagePumpOptions.EmptyQueueMaxFetchDelay) delay = _messagePumpOptions.EmptyQueueMaxFetchDelay; _queueManagers[queueName] = (queueInfo.Config, queueInfo.QueueManager, queueInfo.PoisonQueueManager, DateTime.UtcNow, delay); } diff --git a/Source/Picton.Messaging/MessagePumpOptions.cs b/Source/Picton.Messaging/MessagePumpOptions.cs index 2db7db7..c9951d8 100644 --- a/Source/Picton.Messaging/MessagePumpOptions.cs +++ b/Source/Picton.Messaging/MessagePumpOptions.cs @@ -13,6 +13,8 @@ public record MessagePumpOptions private static readonly TimeSpan _defaultFetchMessagesInterval = TimeSpan.FromSeconds(1); private static readonly TimeSpan _defaultCountAzureMessagesInterval = TimeSpan.FromSeconds(5); private static readonly TimeSpan _defaultCountMemoryMessagesInterval = TimeSpan.FromSeconds(5); + private static readonly TimeSpan _defaultEmptyQueueFetchDelay = TimeSpan.FromSeconds(5); + private static readonly TimeSpan _defaultMaxEmptyQueueFetchDelay = TimeSpan.FromSeconds(30); /// /// Initializes a new instance of the class. @@ -92,5 +94,25 @@ public MessagePumpOptions(string connectionString, int? concurrentTasks, QueueCl /// /// This setting is ignored if you don't specify the sytem where metrics are published. public TimeSpan CountMemoryMessagesInterval { get; set; } = _defaultCountMemoryMessagesInterval; + + /// + /// Gets or sets the delay until the next time a given queue is checked for message when it is determined to be empty. + /// This delay is cumulative, which means that it will be doubled is a queue is found to be empty two times in a row, + /// it will be tripled if the queue is empty three times in a row, etc. This delay is capped at . + /// + /// The delay is reset to zero when at lest one messages is found in the queue. + /// + /// The pupose of the delay is to ensure we don't query a given queue too often when we know it to be empty. + /// + /// Default value is 5 seconds. + /// + public TimeSpan EmptyQueueFetchDelay { get; set; } = _defaultEmptyQueueFetchDelay; + + /// + /// Gets or sets the maximum delay until the next time a given queue is checked for message when it is determined to be empty. + /// + /// Default value is 30 seconds. + /// + public TimeSpan EmptyQueueMaxFetchDelay { get; set; } = _defaultMaxEmptyQueueFetchDelay; } } From fb72fa438c5eec593755718cdb070c1f2310b9a8 Mon Sep 17 00:00:00 2001 From: jericho Date: Mon, 29 Jan 2024 10:43:08 -0500 Subject: [PATCH 24/28] Upgrade XUnit nuget packages --- .../Picton.Messaging.UnitTests.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj b/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj index 5bf32ec..25dd8c7 100644 --- a/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj +++ b/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj @@ -18,8 +18,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers From 94e14efb4c787562a05b8fe6ae61b2ea168b6fcb Mon Sep 17 00:00:00 2001 From: jericho Date: Mon, 29 Jan 2024 10:43:46 -0500 Subject: [PATCH 25/28] Formatting --- .../TestsRunner.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs index 03c4f6a..f1659b2 100644 --- a/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs +++ b/Source/Picton.Messaging.IntegrationTests/TestsRunner.cs @@ -105,27 +105,27 @@ private async Task RunAsyncMessagePumpTests(string connectionString, string queu // Configure the message pump Stopwatch sw = null; + var cts = new CancellationTokenSource(); var options = new MessagePumpOptions(connectionString, concurrentTasks, null, null); var messagePump = new AsyncMessagePump(options, _logger, metrics) { OnMessage = (queueName, message, cancellationToken) => { _logger.LogInformation("{messageContent}", message.Content.ToString()); - } - }; - messagePump.AddQueue(queueName, null, TimeSpan.FromMinutes(1), 3); + }, - // Stop the message pump when the queue is empty. - var cts = new CancellationTokenSource(); - messagePump.OnEmpty = cancellationToken => - { - // Stop the timer - if (sw.IsRunning) sw.Stop(); + // Stop the timer and the message pump when the queue is empty. + OnEmpty = cancellationToken => + { + // Stop the timer + if (sw.IsRunning) sw.Stop(); - // Stop the message pump - _logger.LogDebug("Asking the message pump to stop..."); - cts.Cancel(); + // Stop the message pump + _logger.LogDebug("Asking the message pump to stop..."); + cts.Cancel(); + } }; + messagePump.AddQueue(queueName, null, TimeSpan.FromMinutes(1), 3); // Start the message pump sw = Stopwatch.StartNew(); @@ -158,6 +158,7 @@ private async Task RunAsyncMessagePumpWithHandlersTests(string connectionString, var options = new MessagePumpOptions(connectionString, concurrentTasks, null, null); var messagePump = new AsyncMessagePumpWithHandlers(options, _logger, metrics) { + // Stop the timer and the message pump when the queue is empty. OnEmpty = cancellationToken => { // Stop the timer @@ -207,8 +208,7 @@ private async Task RunMultiTenantAsyncMessagePumpTests(string connectionString, { OnMessage = (tenantId, message, cancellationToken) => { - var messageContent = message.Content.ToString(); - _logger.LogInformation("{tenantId} - {messageContent}", tenantId, messageContent); + _logger.LogInformation("{tenantId} - {messageContent}", tenantId, message.Content.ToString()); }, // Stop the timer and the message pump when all tenant queues are empty. From 0573d395673bad4fd53d51e49f6c4f1f4fbbef11 Mon Sep 17 00:00:00 2001 From: jericho Date: Mon, 29 Jan 2024 13:25:28 -0500 Subject: [PATCH 26/28] (GH-36) Tenants should share the same blob storage for large messages --- README.md | 14 +++++++++++--- Source/Picton.Messaging/AsyncMessagePump.cs | 16 +++++++++++++--- .../AsyncMultiTenantMessagePump.cs | 7 ++++++- Source/Picton.Messaging/QueueConfig.cs | 7 +++++-- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6733894..5f4e731 100644 --- a/README.md +++ b/README.md @@ -141,9 +141,17 @@ namespace WorkerRole1 }; // Replace the following samples with the queues you want to monitor - messagePump.AddQueue("myfirstqueue", "myfirstqueue-poison", TimeSpan.FromMinutes(1), 3); - messagePump.AddQueue("mysecondqueue", "mysecondqueue-poison", TimeSpan.FromMinutes(1), 3); - messagePump.AddQueue("mythirdqueue", "mythirdqueue-poison", TimeSpan.FromMinutes(1), 3); + messagePump.AddQueue("queue01", "queue01-poison", TimeSpan.FromMinutes(1), 3, "queue01-oversize-messages"); + messagePump.AddQueue("queue02", "queue02-poison", TimeSpan.FromMinutes(1), 3, "queue02-oversize-messages"); + messagePump.AddQueue("queue03", "queue03-poison", TimeSpan.FromMinutes(1), 3, "queue03-oversize-messages"); + + // Queues can share the same poison queue + messagePump.AddQueue("queue04", "my-poison-queue", TimeSpan.FromMinutes(1), 3, "queue04-oversize-messages"); + messagePump.AddQueue("queue05", "my-poison-queue", TimeSpan.FromMinutes(1), 3, "queue05-oversize-messages"); + + // Queues can also share the same blob storage for messages that exceed the max size + messagePump.AddQueue("queue06", "my-poison-queue", TimeSpan.FromMinutes(1), 3, "large-messages-blob"); + messagePump.AddQueue("queue07", "my-poison-queue", TimeSpan.FromMinutes(1), 3, "large-messages-blob"); // Start the message pump await messagePump.StartAsync(cancellationToken); diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index a16ec99..c522f85 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -1,5 +1,7 @@ using App.Metrics; using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Queues; using Microsoft.Extensions.Logging; using Picton.Managers; using Picton.Messaging.Utilities; @@ -107,9 +109,10 @@ public AsyncMessagePump(MessagePumpOptions options, ILogger logger = null, IMetr /// Optional. The name of the queue where poison messages are automatically moved. /// Optional. Specifies the visibility timeout value. The default value is 30 seconds. /// Optional. A nonzero integer value that specifies the number of time we try to process a message before giving up and declaring the message to be "poison". The default value is 3. - public void AddQueue(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3) + /// Name of the blob storage where messages that exceed the maximum size for a queue message are stored. + public void AddQueue(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, string oversizeMessagesBlobStorageName = null) { - AddQueue(new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount)); + AddQueue(new QueueConfig(queueName, poisonQueueName, visibilityTimeout, maxDequeueCount, oversizeMessagesBlobStorageName)); } /// @@ -121,7 +124,14 @@ public void AddQueue(QueueConfig queueConfig) if (string.IsNullOrEmpty(queueConfig.QueueName)) throw new ArgumentNullException(nameof(queueConfig.QueueName)); if (queueConfig.MaxDequeueCount < 1) throw new ArgumentOutOfRangeException(nameof(queueConfig.MaxDequeueCount), "Number of retries must be greater than zero."); - var queueManager = new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.QueueName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); + // ---------------------------------------------------------------------------------------------------- + // When Picton 9.2.0 is released, when can replace the following few lines with the following single line: + // var queueManager = new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.QueueName, queueConfig.OversizedMessagesBlobStorageName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); + var blobStorageName = string.IsNullOrEmpty(queueConfig.OversizedMessagesBlobStorageName) ? $"{queueConfig.QueueName}-oversize-messages" : queueConfig.OversizedMessagesBlobStorageName; + var blobClient = new BlobContainerClient(_messagePumpOptions.ConnectionString, blobStorageName, _messagePumpOptions.BlobClientOptions); + var queueClient = new QueueClient(_messagePumpOptions.ConnectionString, queueConfig.QueueName, _messagePumpOptions.QueueClientOptions); + var queueManager = new QueueManager(blobClient, queueClient, true); + var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.PoisonQueueName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); AddQueue(queueManager, poisonQueueManager, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); diff --git a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs index acd100b..8d6a1c8 100644 --- a/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMultiTenantMessagePump.cs @@ -138,7 +138,12 @@ public async Task StartAsync(CancellationToken cancellationToken) !queue.Name.Equals($"{_queueNamePrefix}-poison", StringComparison.OrdinalIgnoreCase)) { // AddQueue will make sure to add the queue only if it's not already in the round-robin list of queues. - _messagePump.AddQueue(queue.Name, $"{_queueNamePrefix}-poison", _visibilityTimeout, _maxDequeueCount); + _messagePump.AddQueue( + queue.Name, + $"{_queueNamePrefix}-poison", // All tenants share the same "poison" queue + _visibilityTimeout, + _maxDequeueCount, + $"{_queueNamePrefix}-oversize-messages"); // All tenants share the same "oversize messages" blob storage } } } diff --git a/Source/Picton.Messaging/QueueConfig.cs b/Source/Picton.Messaging/QueueConfig.cs index 2487968..7a4ad94 100644 --- a/Source/Picton.Messaging/QueueConfig.cs +++ b/Source/Picton.Messaging/QueueConfig.cs @@ -4,18 +4,21 @@ namespace Picton.Messaging { public record QueueConfig { - public QueueConfig(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3) + public QueueConfig(string queueName, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, string oversizedMessagesBlobStorageName = null) { QueueName = queueName ?? throw new ArgumentNullException(nameof(queueName)); PoisonQueueName = poisonQueueName; VisibilityTimeout = visibilityTimeout; MaxDequeueCount = maxDequeueCount; + OversizedMessagesBlobStorageName = oversizedMessagesBlobStorageName; } - public string QueueName { get; set; } + public string QueueName { get; set; } = null; public string PoisonQueueName { get; set; } = null; + public string OversizedMessagesBlobStorageName { get; set; } = null; + public TimeSpan? VisibilityTimeout { get; set; } = null; public int MaxDequeueCount { get; set; } = 3; From fc3c30102dd7e1e16e0cba8078637ecd2063237f Mon Sep 17 00:00:00 2001 From: jericho Date: Mon, 29 Jan 2024 16:34:03 -0500 Subject: [PATCH 27/28] (GH-36) Tenants should share the same blob storage for large messages --- Source/Picton.Messaging/AsyncMessagePump.cs | 13 ++----------- Source/Picton.Messaging/Picton.Messaging.csproj | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index c522f85..c4f4493 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -1,7 +1,5 @@ using App.Metrics; using Azure; -using Azure.Storage.Blobs; -using Azure.Storage.Queues; using Microsoft.Extensions.Logging; using Picton.Managers; using Picton.Messaging.Utilities; @@ -124,15 +122,8 @@ public void AddQueue(QueueConfig queueConfig) if (string.IsNullOrEmpty(queueConfig.QueueName)) throw new ArgumentNullException(nameof(queueConfig.QueueName)); if (queueConfig.MaxDequeueCount < 1) throw new ArgumentOutOfRangeException(nameof(queueConfig.MaxDequeueCount), "Number of retries must be greater than zero."); - // ---------------------------------------------------------------------------------------------------- - // When Picton 9.2.0 is released, when can replace the following few lines with the following single line: - // var queueManager = new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.QueueName, queueConfig.OversizedMessagesBlobStorageName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); - var blobStorageName = string.IsNullOrEmpty(queueConfig.OversizedMessagesBlobStorageName) ? $"{queueConfig.QueueName}-oversize-messages" : queueConfig.OversizedMessagesBlobStorageName; - var blobClient = new BlobContainerClient(_messagePumpOptions.ConnectionString, blobStorageName, _messagePumpOptions.BlobClientOptions); - var queueClient = new QueueClient(_messagePumpOptions.ConnectionString, queueConfig.QueueName, _messagePumpOptions.QueueClientOptions); - var queueManager = new QueueManager(blobClient, queueClient, true); - - var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.PoisonQueueName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); + var queueManager = new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.QueueName, queueConfig.OversizedMessagesBlobStorageName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); + var poisonQueueManager = string.IsNullOrEmpty(queueConfig.PoisonQueueName) ? null : new QueueManager(_messagePumpOptions.ConnectionString, queueConfig.PoisonQueueName, queueConfig.OversizedMessagesBlobStorageName, true, _messagePumpOptions.QueueClientOptions, _messagePumpOptions.BlobClientOptions); AddQueue(queueManager, poisonQueueManager, queueConfig.VisibilityTimeout, queueConfig.MaxDequeueCount); } diff --git a/Source/Picton.Messaging/Picton.Messaging.csproj b/Source/Picton.Messaging/Picton.Messaging.csproj index b0a7d59..cf46aa4 100644 --- a/Source/Picton.Messaging/Picton.Messaging.csproj +++ b/Source/Picton.Messaging/Picton.Messaging.csproj @@ -40,7 +40,7 @@ - + From f7f845d642f82eda4817c7269a671d1721faa519 Mon Sep 17 00:00:00 2001 From: jericho Date: Mon, 29 Jan 2024 16:36:29 -0500 Subject: [PATCH 28/28] Add multiple queue that match a RegEx pattern Resolves #37 --- Source/Picton.Messaging/AsyncMessagePump.cs | 44 +++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index c4f4493..046911d 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -1,13 +1,17 @@ using App.Metrics; using Azure; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; using Microsoft.Extensions.Logging; using Picton.Managers; using Picton.Messaging.Utilities; using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -142,6 +146,46 @@ public void RemoveQueue(string queueName) _queueNames.RemoveItem(queueName); } + /// + /// Add queues that meet the specified RegEx pattern. + /// + /// + /// All the queues that match the specified pattern will share the same poison queue if you specify the name of the poison queue. + /// If you omit this value, each queue will get their own poison queue. + /// + /// Similarly, all the queues that match the specified pattern will share the same oversize messages storage if you specify the name of the blob storage container. + /// If you omit this value, each queue will get their own blob container. + /// + /// The RegEx pattern. + /// Optional. The name of the queue where poison messages are automatically moved. + /// Optional. Specifies the visibility timeout value. The default value is 30 seconds. + /// Optional. A nonzero integer value that specifies the number of time we try to process a message before giving up and declaring the message to be "poison". The default value is 3. + /// Name of the blob storage where messages that exceed the maximum size for a queue message are stored. + /// The cancellation token. + /// The async task. + public async Task AddQueuesByPatternAsync(string queueNamePattern, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, string oversizeMessagesBlobStorageName = null, CancellationToken cancellationToken = default) + { + var regex = new Regex(queueNamePattern, RegexOptions.Compiled); + var queueServiceClient = new QueueServiceClient(_messagePumpOptions.ConnectionString); + var response = queueServiceClient.GetQueuesAsync(QueueTraits.None, null, cancellationToken); + await foreach (Page queues in response.AsPages()) + { + foreach (var queue in queues.Values) + { + if (regex.IsMatch(queue.Name)) + { + AddQueue(new QueueConfig(queue.Name, poisonQueueName, visibilityTimeout, maxDequeueCount, oversizeMessagesBlobStorageName)); + } + } + } + } + + /// + /// Gets the names of the queues currently being monitored. + /// + /// The read only list of queue names. + public IReadOnlyList GetMonitoredQueueNames() => ImmutableList.CreateRange(_queueManagers.Keys); + /// /// Starts the message pump. ///