Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,36 @@ public class RpcCallRetryHandlerFixture
{
readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2);

//[Test]
public async Task? WhatDoesCancelledDo()
{
var expectedResult = Guid.NewGuid();

var handler = new RpcCallRetryHandler(TimeSpan.FromSeconds(60));

int count = 0;

var result = await handler.ExecuteWithRetries(
async ct =>
{
await Task.CompletedTask;

count++;
if (count == 5)
{
return expectedResult;
}

throw new Exception();
},
onRetryAction: null,
onTimeoutAction: null,
CancellationToken.None);

result.Should().Be(expectedResult);
}


[Test]
public async Task ReturnsTheResultWhenNoRetries()
{
Expand Down
3 changes: 3 additions & 0 deletions source/Octopus.Tentacle.Client/Execution/RpcCallExecutor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public async Task<T> ExecuteWithRetries<T>(
}
catch (Exception e)
{
logger.Warn(e, $"Error occured talking to tentacle, Is given CancellationToken cancelled? {cancellationToken.IsCancellationRequested} Is inner CT cancelled {ct.IsCancellationRequested}");
rpcCallMetricsBuilder.WithAttempt(TimedOperation.Failure(start, e, ct));
throw;
}
Expand Down Expand Up @@ -101,12 +102,14 @@ public async Task<T> ExecuteWithRetries<T>(
logger.Info($"Could not communicate with Tentacle after {(int)elapsedDuration.TotalSeconds} seconds.");
}
},
logger,
cancellationToken);

return response;
}
catch (Exception e)
{
logger.Warn(e, "Some error occured talking to tentacle");
rpcCallMetricsBuilder.Failure(e, cancellationToken);
throw;
}
Expand Down
15 changes: 15 additions & 0 deletions source/Octopus.Tentacle.Client/Retries/RpcCallRetryHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Threading;
using System.Threading.Tasks;
using Octopus.Tentacle.Client.Scripts;
using Octopus.Tentacle.Contracts.Logging;
using Polly;
using Polly.Timeout;

Expand Down Expand Up @@ -38,6 +39,16 @@ public async Task<T> ExecuteWithRetries<T>(
OnRetryAction? onRetryAction,
OnTimeoutAction? onTimeoutAction,
CancellationToken cancellationToken)
{
return await ExecuteWithRetries<T>(action, onRetryAction, onTimeoutAction, null, cancellationToken);
}

public async Task<T> ExecuteWithRetries<T>(
Func<CancellationToken, Task<T>> action,
OnRetryAction? onRetryAction,
OnTimeoutAction? onTimeoutAction,
ITentacleClientTaskLog? logger,
CancellationToken cancellationToken)
{
Exception? lastException = null;
var started = new Stopwatch();
Expand Down Expand Up @@ -71,6 +82,10 @@ async Task OnRetryAction(Exception exception, TimeSpan sleepDuration, int retryC
await onRetryAction.Invoke(exception, sleepDuration, retryCount, RetryTimeout, elapsedDuration, cancellationToken).ConfigureAwait(false);
}
}
else
{
logger?.Info("Decided not to retry");
}
}

async Task OnTimeoutAction(Context? context, TimeSpan timeout, Task? task, Exception? exception)
Expand Down