feat: Add exception handling and re-queueing to finalizers (#244)

Author: erin-allison

src/KubeOps/Operator/Controller/ManagedResourceController{TEntity}.cs

@@ -287,12 +287,28 @@ protected async Task HandleResourceFinalization(QueuedEvent? data)
                 return;
             }
 
+            var (_, resource, retryCount) = data;
+
             _logger.LogDebug(
                 @"Finalize resource ""{kind}/{name}"".",
-                data.Resource.Kind,
-                data.Resource.Name());
+                resource.Kind,
+                resource.Name());
 
-            await _finalizerManager.FinalizeAsync(data.Resource);
+            try
+            {
+                await _finalizerManager.FinalizeAsync(data.Resource);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(
+                    e,
+                    @"Finalize resource ""{kind}/{name}"" threw an error. Retry attempt {retryAttempt}.",
+                    resource.Kind,
+                    resource.Name(),
+                    retryCount + 1);
+                _erroredEvents.OnNext(data with { RetryCount = retryCount + 1 });
+                return;
+            }
         }
 
         private (ResourceEventType ResourceEvent, TEntity Resource) MapCacheResult(