Skip to content

Commit fe27faa

Browse files
borlandborland
authored
Merge pull request #1148 from OctopusDeploy/orion/add-tracing
Adding OTEL Traces to TentacleClient
2 parents 974f475 + f98f16e commit fe27faa

File tree

7 files changed

+68
-0
lines changed

7 files changed

+68
-0
lines changed

‎source/Octopus.Tentacle.Client/Execution/RpcCallExecutor.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Diagnostics;
23
using System.Threading;
34
using System.Threading.Tasks;
45
using Octopus.Tentacle.Client.Observability;
@@ -56,6 +57,9 @@ public async Task<T> ExecuteWithRetries<T>(
5657
ClientOperationMetricsBuilder clientOperationMetricsBuilder,
5758
CancellationToken cancellationToken)
5859
{
60+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(RpcCallExecutor)}.{nameof(ExecuteWithRetries)}");
61+
activity?.AddTag("octopus.tentacle.rpc_call.service", rpcCall.Service);
62+
activity?.AddTag("octopus.tentacle.rpc_call.name", rpcCall.Name);
5963
var rpcCallMetricsBuilder = RpcCallMetricsBuilder.StartWithRetries(rpcCall, rpcCallRetryHandler.RetryTimeout);
6064

6165
try
@@ -107,6 +111,10 @@ public async Task<T> ExecuteWithRetries<T>(
107111
}
108112
catch (Exception e)
109113
{
114+
activity?.SetStatus(ActivityStatusCode.Error);
115+
// We should use activity.AddException here, but need to update the referenced version of System.Diagnostics.DiagnosticSource.
116+
// We inherit the reference from Halibut so that would have to change first.
117+
activity?.AddTag("exception.message", e.Message);
110118
rpcCallMetricsBuilder.Failure(e, cancellationToken);
111119
throw;
112120
}
@@ -125,6 +133,9 @@ public async Task<T> ExecuteWithNoRetries<T>(
125133
ClientOperationMetricsBuilder clientOperationMetricsBuilder,
126134
CancellationToken cancellationToken)
127135
{
136+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(RpcCallExecutor)}.{nameof(ExecuteWithNoRetries)}");
137+
activity?.AddTag("octopus.tentacle.rpc_call.service", rpcCall.Service);
138+
activity?.AddTag("octopus.tentacle.rpc_call.name", rpcCall.Name);
128139
var rpcCallMetricsBuilder = RpcCallMetricsBuilder.StartWithoutRetries(rpcCall);
129140
var start = DateTimeOffset.UtcNow;
130141

@@ -136,6 +147,11 @@ public async Task<T> ExecuteWithNoRetries<T>(
136147
}
137148
catch (Exception e)
138149
{
150+
activity?.SetStatus(ActivityStatusCode.Error);
151+
// We should use activity.AddException here, but need to update the referenced version of System.Diagnostics.DiagnosticSource.
152+
// We inherit the reference from Halibut so that would have to change first.
153+
activity?.AddTag("exception.message", e.Message);
154+
139155
rpcCallMetricsBuilder.WithAttempt(TimedOperation.Failure(start, e, cancellationToken));
140156
rpcCallMetricsBuilder.Failure(e, cancellationToken);
141157
throw;
@@ -156,6 +172,9 @@ public async Task ExecuteWithNoRetries(
156172
ClientOperationMetricsBuilder clientOperationMetricsBuilder,
157173
CancellationToken cancellationToken)
158174
{
175+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(RpcCallExecutor)}.{nameof(ExecuteWithNoRetries)}");
176+
activity?.AddTag("octopus.tentacle.rpc_call.service", rpcCall.Service);
177+
activity?.AddTag("octopus.tentacle.rpc_call.name", rpcCall.Name);
159178
var rpcCallMetricsBuilder = RpcCallMetricsBuilder.StartWithoutRetries(rpcCall);
160179
var start = DateTimeOffset.UtcNow;
161180

@@ -166,6 +185,10 @@ public async Task ExecuteWithNoRetries(
166185
}
167186
catch (Exception e)
168187
{
188+
activity?.SetStatus(ActivityStatusCode.Error);
189+
// We should use activity.AddException here, but need to update the referenced version of System.Diagnostics.DiagnosticSource.
190+
// We inherit the reference from Halibut so that would have to change first.
191+
activity?.AddTag("exception.message", e.Message);
169192
rpcCallMetricsBuilder.WithAttempt(TimedOperation.Failure(start, e, cancellationToken));
170193
rpcCallMetricsBuilder.Failure(e, cancellationToken);
171194
throw;

‎source/Octopus.Tentacle.Client/ScriptExecutor.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ public async Task<ScriptOperationExecutionResult> StartScript(ExecuteScriptComma
4444
StartScriptIsBeingReAttempted startScriptIsBeingReAttempted,
4545
CancellationToken cancellationToken)
4646
{
47+
// Note: This class deliberately does not create OpenTelemetry Trace activities.
48+
// It is a facade over other ScriptExecutor services, and the facade doesn't do anything interesting
4749
var scriptServiceVersionToUse = await DetermineScriptServiceVersionToUse(cancellationToken);
4850

4951
var scriptExecutorFactory = CreateScriptExecutorFactory();

‎source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1Executor.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ public async Task<ScriptOperationExecutionResult> StartScript(ExecuteScriptComma
7979
StartScriptIsBeingReAttempted startScriptIsBeingReAttempted,
8080
CancellationToken scriptExecutionCancellationToken)
8181
{
82+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(KubernetesScriptServiceV1Executor)}.{nameof(StartScript)}");
83+
8284
var command = Map(executeScriptCommand);
8385
var startScriptCallsConnectedCount = 0;
8486
try
@@ -135,6 +137,7 @@ void OnErrorAction(Exception ex)
135137

136138
public async Task<ScriptOperationExecutionResult> GetStatus(CommandContext commandContext, CancellationToken scriptExecutionCancellationToken)
137139
{
140+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(KubernetesScriptServiceV1Executor)}.{nameof(GetStatus)}");
138141
async Task<KubernetesScriptStatusResponseV1> GetStatusAction(CancellationToken ct)
139142
{
140143
var request = new KubernetesScriptStatusRequestV1(commandContext.ScriptTicket, commandContext.NextLogSequence);
@@ -155,6 +158,7 @@ async Task<KubernetesScriptStatusResponseV1> GetStatusAction(CancellationToken c
155158

156159
public async Task<ScriptOperationExecutionResult> CancelScript(CommandContext commandContext)
157160
{
161+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(KubernetesScriptServiceV1Executor)}.{nameof(CancelScript)}");
158162
async Task<KubernetesScriptStatusResponseV1> CancelScriptAction(CancellationToken ct)
159163
{
160164
var request = new CancelKubernetesScriptCommandV1(commandContext.ScriptTicket, commandContext.NextLogSequence);
@@ -180,6 +184,7 @@ async Task<KubernetesScriptStatusResponseV1> CancelScriptAction(CancellationToke
180184

181185
public async Task<ScriptStatus?> CompleteScript(CommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken)
182186
{
187+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(KubernetesScriptServiceV1Executor)}.{nameof(CompleteScript)}");
183188
try
184189
{
185190
// Finish performs a best effort cleanup of the Workspace on Tentacle

‎source/Octopus.Tentacle.Client/Scripts/ObservingScriptOrchestrator.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Diagnostics;
23
using System.Threading;
34
using System.Threading.Tasks;
45
using Halibut.Logging;
@@ -47,6 +48,10 @@ async Task<ScriptStatus> ObserveUntilCompleteThenFinish(
4748
ScriptOperationExecutionResult startScriptResult,
4849
CancellationToken scriptExecutionCancellationToken)
4950
{
51+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ObservingScriptOrchestrator)}.{nameof(ObserveUntilCompleteThenFinish)}");
52+
activity?.AddTag("octopus.tentacle.script.status.state", startScriptResult.ScriptStatus.State);
53+
activity?.AddTag("octopus.tentacle.script.status.exit_code", startScriptResult.ScriptStatus.ExitCode);
54+
5055
OnScriptStatusResponseReceived(startScriptResult.ScriptStatus);
5156

5257
var observingUntilCompleteResult = await ObserveUntilComplete(startScriptResult, scriptExecutionCancellationToken).ConfigureAwait(false);

‎source/Octopus.Tentacle.Client/Scripts/ScriptServiceV1Executor.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ public async Task<ScriptOperationExecutionResult> StartScript(ExecuteScriptComma
8282
StartScriptIsBeingReAttempted startScriptIsBeingReAttempted,
8383
CancellationToken scriptExecutionCancellationToken)
8484
{
85+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ScriptServiceV1Executor)}.{nameof(StartScript)}");
86+
8587
// Script Service v1 is not idempotent, do not allow it to be re-attempted as it may run a second time.
8688
if (startScriptIsBeingReAttempted == StartScriptIsBeingReAttempted.PossiblyBeingReAttempted)
8789
{
@@ -106,6 +108,7 @@ public async Task<ScriptOperationExecutionResult> StartScript(ExecuteScriptComma
106108

107109
public async Task<ScriptOperationExecutionResult> GetStatus(CommandContext commandContext, CancellationToken scriptExecutionCancellationToken)
108110
{
111+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ScriptServiceV1Executor)}.{nameof(GetStatus)}");
109112
var scriptStatusResponseV1 = await GetStatusV1(commandContext, scriptExecutionCancellationToken).ConfigureAwait(false);
110113

111114
if (scriptStatusResponseV1.State != ProcessState.Complete)
@@ -125,6 +128,7 @@ public async Task<ScriptOperationExecutionResult> GetStatus(CommandContext comma
125128

126129
async Task<ScriptStatusResponse> GetStatusV1(CommandContext commandContext, CancellationToken scriptExecutionCancellationToken)
127130
{
131+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ScriptServiceV1Executor)}.{nameof(GetStatusV1)}");
128132
var scriptStatusResponseV1 = await rpcCallExecutor.ExecuteWithNoRetries(
129133
RpcCall.Create<IScriptService>(nameof(IScriptService.GetStatus)),
130134
async ct =>
@@ -142,6 +146,7 @@ async Task<ScriptStatusResponse> GetStatusV1(CommandContext commandContext, Canc
142146

143147
public async Task<ScriptOperationExecutionResult> CancelScript(CommandContext commandContext)
144148
{
149+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ScriptServiceV1Executor)}.{nameof(CancelScript)}");
145150
var response = await rpcCallExecutor.ExecuteWithNoRetries(
146151
RpcCall.Create<IScriptService>(nameof(IScriptService.CancelScript)),
147152
async ct =>
@@ -160,6 +165,7 @@ public async Task<ScriptOperationExecutionResult> CancelScript(CommandContext co
160165

161166
public async Task<ScriptStatus?> CompleteScript(CommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken)
162167
{
168+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ScriptServiceV1Executor)}.{nameof(CompleteScript)}");
163169
var response = await rpcCallExecutor.ExecuteWithNoRetries(
164170
RpcCall.Create<IScriptService>(nameof(IScriptService.CompleteScript)),
165171
async ct =>

‎source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ void OnErrorAction(Exception ex)
128128

129129
public async Task<ScriptOperationExecutionResult> GetStatus(CommandContext commandContext, CancellationToken scriptExecutionCancellationToken)
130130
{
131+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ScriptServiceV2Executor)}.{nameof(GetStatus)}");
131132
async Task<ScriptStatusResponseV2> GetStatusAction(CancellationToken ct)
132133
{
133134
var request = new ScriptStatusRequestV2(commandContext.ScriptTicket, commandContext.NextLogSequence);
@@ -148,6 +149,7 @@ async Task<ScriptStatusResponseV2> GetStatusAction(CancellationToken ct)
148149

149150
public async Task<ScriptOperationExecutionResult> CancelScript(CommandContext commandContext)
150151
{
152+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ScriptServiceV2Executor)}.{nameof(CancelScript)}");
151153
async Task<ScriptStatusResponseV2> CancelScriptAction(CancellationToken ct)
152154
{
153155
var request = new CancelScriptCommandV2(commandContext.ScriptTicket, commandContext.NextLogSequence);
@@ -173,6 +175,7 @@ async Task<ScriptStatusResponseV2> CancelScriptAction(CancellationToken ct)
173175

174176
public async Task<ScriptStatus?> CompleteScript(CommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken)
175177
{
178+
using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(ScriptServiceV2Executor)}.{nameof(CompleteScript)}");
176179
try
177180
{
178181
// Finish performs a best effort cleanup of the Workspace on Tentacle

‎source/Octopus.Tentacle.Client/TentacleClient.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using System;
2+
using System.Diagnostics;
23
using System.IO;
4+
using System.Linq;
35
using System.Threading;
46
using System.Threading.Tasks;
57
using Halibut;
@@ -20,6 +22,8 @@ namespace Octopus.Tentacle.Client
2022
{
2123
public class TentacleClient : ITentacleClient
2224
{
25+
public static readonly ActivitySource ActivitySource = new("Octopus.TentacleClient");
26+
2327
readonly IScriptObserverBackoffStrategy scriptObserverBackOffStrategy;
2428
readonly ITentacleClientObserver tentacleClientObserver;
2529
readonly RpcCallExecutor rpcCallExecutor;
@@ -29,6 +33,8 @@ public class TentacleClient : ITentacleClient
2933

3034
public static void CacheServiceWasNotFoundResponseMessages(IHalibutRuntime halibutRuntime)
3135
{
36+
using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(CacheServiceWasNotFoundResponseMessages)}");
37+
3238
var innerHandler = halibutRuntime.OverrideErrorResponseMessageCaching;
3339
halibutRuntime.OverrideErrorResponseMessageCaching = response =>
3440
{
@@ -85,6 +91,9 @@ internal TentacleClient(
8591

8692
public async Task<UploadResult> UploadFile(string fileName, string path, DataStream package, ITentacleClientTaskLog logger, CancellationToken cancellationToken)
8793
{
94+
using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(UploadFile)}");
95+
activity?.AddTag("octopus.tentacle.file_name", fileName);
96+
activity?.AddTag("octopus.tentacle.file_path", path);
8897
var operationMetricsBuilder = ClientOperationMetricsBuilder.Start();
8998

9099
async Task<UploadResult> UploadFileAction(CancellationToken ct)
@@ -120,6 +129,8 @@ async Task<UploadResult> UploadFileAction(CancellationToken ct)
120129

121130
public async Task<DataStream?> DownloadFile(string remotePath, ITentacleClientTaskLog logger, CancellationToken cancellationToken)
122131
{
132+
using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(DownloadFile)}");
133+
activity?.AddTag("octopus.tentacle.remote_path", remotePath);
123134
var operationMetricsBuilder = ClientOperationMetricsBuilder.Start();
124135

125136
async Task<DataStream> DownloadFileAction(CancellationToken ct)
@@ -159,6 +170,9 @@ public async Task<ScriptExecutionResult> ExecuteScript(ExecuteScriptCommand exec
159170
ITentacleClientTaskLog logger,
160171
CancellationToken scriptExecutionCancellationToken)
161172
{
173+
using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(ExecuteScript)}");
174+
activity?.AddTag("octopus.tentacle.script.files", string.Join(",", executeScriptCommand.Files.Select(f => f.Name)));
175+
// don't trace the script body, it could be megabytes in size
162176
var operationMetricsBuilder = ClientOperationMetricsBuilder.Start();
163177

164178
try
@@ -198,6 +212,10 @@ public async Task<ScriptOperationExecutionResult> StartScript(
198212
ITentacleClientTaskLog logger,
199213
CancellationToken scriptExecutionCancellationToken)
200214
{
215+
using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(StartScript)}");
216+
activity?.AddTag("octopus.tentacle.script.files", string.Join(",", command.Files.Select(f => f.Name)));
217+
// don't trace the script body, it could be megabytes in size
218+
201219
var scriptExecutor = new ScriptExecutor(
202220
allClients,
203221
logger,
@@ -212,6 +230,8 @@ public async Task<ScriptOperationExecutionResult> StartScript(
212230

213231
public async Task<ScriptOperationExecutionResult> GetStatus(CommandContext commandContext, ITentacleClientTaskLog logger, CancellationToken scriptExecutionCancellationToken)
214232
{
233+
using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(GetStatus)}");
234+
215235
var scriptExecutor = new ScriptExecutor(
216236
allClients,
217237
logger,
@@ -226,6 +246,8 @@ public async Task<ScriptOperationExecutionResult> GetStatus(CommandContext comma
226246

227247
public async Task<ScriptOperationExecutionResult> CancelScript(CommandContext commandContext, ITentacleClientTaskLog logger)
228248
{
249+
using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(CancelScript)}");
250+
229251
var scriptExecutor = new ScriptExecutor(
230252
allClients,
231253
logger,
@@ -240,6 +262,8 @@ public async Task<ScriptOperationExecutionResult> CancelScript(CommandContext co
240262

241263
public async Task<ScriptStatus?> CompleteScript(CommandContext commandContext, ITentacleClientTaskLog logger, CancellationToken scriptExecutionCancellationToken)
242264
{
265+
using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(CompleteScript)}");
266+
243267
var scriptExecutor = new ScriptExecutor(
244268
allClients,
245269
logger,

0 commit comments

Comments
 (0)