diff --git a/CHANGELOG.md b/CHANGELOG.md index 67608bc13f..51f85b862b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Features + +- Added a new SDK `Sentry.Extensions.AI` which allows LLM usage instrumentation via `Microsoft.Extensions.AI` ([#4657](https://github.com/getsentry/sentry-dotnet/pull/4657)) + ### Fixes - Captured [Http Client Errors](https://docs.sentry.io/platforms/dotnet/guides/aspnet/configuration/http-client-errors/) on .NET 5+ now include a full stack trace in order to improve Issue grouping ([#4724](https://github.com/getsentry/sentry-dotnet/pull/4724)) @@ -47,7 +51,6 @@ - The SDK now makes use of the new SessionEndStatus `Unhandled` when capturing an unhandled but non-terminal exception, i.e. through the UnobservedTaskExceptionIntegration ([#4633](https://github.com/getsentry/sentry-dotnet/pull/4633), [#4653](https://github.com/getsentry/sentry-dotnet/pull/4653)) - Implemented instance isolation so that multiple instances of the Sentry SDK can be instantiated inside the same process when using the Caching Transport ([#4498](https://github.com/getsentry/sentry-dotnet/pull/4498)) - Extended the App context by `app_memory` that can hold the amount of memory used by the application in bytes. ([#4707](https://github.com/getsentry/sentry-dotnet/pull/4707)) -- Added a new SDK `Sentry.Extensions.AI` which allows LLM usage instrumentation via `Microsoft.Extensions.AI` ([#4657](https://github.com/getsentry/sentry-dotnet/pull/4657)) ### Fixes diff --git a/samples/Sentry.Samples.ME.AI.AspNetCore/Program.cs b/samples/Sentry.Samples.ME.AI.AspNetCore/Program.cs index 83f8731f28..d20512aa33 100644 --- a/samples/Sentry.Samples.ME.AI.AspNetCore/Program.cs +++ b/samples/Sentry.Samples.ME.AI.AspNetCore/Program.cs @@ -1,4 +1,3 @@ -#nullable enable using Microsoft.Extensions.AI; var builder = WebApplication.CreateBuilder(args); diff --git a/samples/Sentry.Samples.ME.AI.AspNetCore/Sentry.Samples.ME.AI.AspNetCore.csproj b/samples/Sentry.Samples.ME.AI.AspNetCore/Sentry.Samples.ME.AI.AspNetCore.csproj index e4c8c3a6d7..aa0b206919 100644 --- a/samples/Sentry.Samples.ME.AI.AspNetCore/Sentry.Samples.ME.AI.AspNetCore.csproj +++ b/samples/Sentry.Samples.ME.AI.AspNetCore/Sentry.Samples.ME.AI.AspNetCore.csproj @@ -1,7 +1,8 @@  - net9.0 + net10.0 + enable diff --git a/samples/Sentry.Samples.ME.AI.Console/Program.cs b/samples/Sentry.Samples.ME.AI.Console/Program.cs index 03883b3663..74e2a6321d 100644 --- a/samples/Sentry.Samples.ME.AI.Console/Program.cs +++ b/samples/Sentry.Samples.ME.AI.Console/Program.cs @@ -96,11 +96,8 @@ "Please help me with the following tasks: 1) Find Alice's age, 2) Get weather in New York, and 3) Calculate a complex result for number 15. Please use the appropriate tools for each task.", options); -logger.LogInformation("Response: {ResponseText}", response.Messages?.FirstOrDefault()?.Text ?? "No response"); +logger.LogInformation("Response: {ResponseText}", response.Messages.FirstOrDefault()?.Text ?? "No response"); logger.LogInformation("Microsoft.Extensions.AI sample completed! Check your Sentry dashboard for the trace data."); transaction.Finish(); - -// Flush Sentry to ensure all transactions are sent before the app exits -await SentrySdk.FlushAsync(TimeSpan.FromSeconds(2)); diff --git a/samples/Sentry.Samples.ME.AI.Console/Sentry.Samples.ME.AI.Console.csproj b/samples/Sentry.Samples.ME.AI.Console/Sentry.Samples.ME.AI.Console.csproj index 39b28ca53d..b1d5683a52 100644 --- a/samples/Sentry.Samples.ME.AI.Console/Sentry.Samples.ME.AI.Console.csproj +++ b/samples/Sentry.Samples.ME.AI.Console/Sentry.Samples.ME.AI.Console.csproj @@ -2,7 +2,8 @@ Exe - net9.0 + net10.0 + enable diff --git a/src/Sentry.Extensions.AI/Sentry.Extensions.AI.csproj b/src/Sentry.Extensions.AI/Sentry.Extensions.AI.csproj index 95754ae836..ea6c6f93c7 100644 --- a/src/Sentry.Extensions.AI/Sentry.Extensions.AI.csproj +++ b/src/Sentry.Extensions.AI/Sentry.Extensions.AI.csproj @@ -3,7 +3,7 @@ $(CurrentTfms);netstandard2.0 $(PackageTags);Microsoft.Extensions.AI;AI;LLM - Microsoft.Extensions.AI integration for Sentry - captures AI Agent telemetry spans per Sentry AI Agents module. + Official Microsoft.Extensions.AI integration for Sentry - Open-source error tracking that helps developers monitor and fix crashes in real time. diff --git a/src/Sentry.Extensions.AI/SentryChatClient.cs b/src/Sentry.Extensions.AI/SentryChatClient.cs index fd8b81faac..35fe06d3a6 100644 --- a/src/Sentry.Extensions.AI/SentryChatClient.cs +++ b/src/Sentry.Extensions.AI/SentryChatClient.cs @@ -5,19 +5,20 @@ namespace Sentry.Extensions.AI; internal sealed class SentryChatClient : DelegatingChatClient { private readonly ActivitySource _activitySource; - private readonly HubAdapter _hub = HubAdapter.Instance; + private readonly IHub _hub; private readonly SentryAIOptions _sentryAIOptions; - public SentryChatClient(IChatClient client, Action? configure = null) : this(null, client, configure) + public SentryChatClient(IChatClient client, Action? configure = null) : this(null, null, client, configure) { } /// - /// Internal ovverride for testing + /// Internal overload for testing /// - internal SentryChatClient(ActivitySource? activitySource, IChatClient client, Action? configure = null) : base(client) + internal SentryChatClient(ActivitySource? activitySource, IHub? hub, IChatClient client, Action? configure = null) : base(client) { _activitySource = activitySource ?? SentryAIActivitySource.Instance; + _hub = hub ?? HubAdapter.Instance; _sentryAIOptions = new SentryAIOptions(); configure?.Invoke(_sentryAIOptions); } diff --git a/src/Sentry.Extensions.AI/SentryInstrumentedFunction.cs b/src/Sentry.Extensions.AI/SentryInstrumentedFunction.cs index 09ae82b60e..e6133fba24 100644 --- a/src/Sentry.Extensions.AI/SentryInstrumentedFunction.cs +++ b/src/Sentry.Extensions.AI/SentryInstrumentedFunction.cs @@ -6,6 +6,7 @@ internal sealed class SentryInstrumentedFunction(AIFunction innerFunction, IHub? : DelegatingAIFunction(innerFunction) { private readonly IHub _hub = hub ?? HubAdapter.Instance; + protected override async ValueTask InvokeCoreAsync( AIFunctionArguments arguments, CancellationToken cancellationToken) diff --git a/test/Sentry.Extensions.AI.Tests/ApiApprovalTests.Run.Net4_8.verified.txt b/test/Sentry.Extensions.AI.Tests/ApiApprovalTests.Run.Net4_8.verified.txt new file mode 100644 index 0000000000..2e54508025 --- /dev/null +++ b/test/Sentry.Extensions.AI.Tests/ApiApprovalTests.Run.Net4_8.verified.txt @@ -0,0 +1,23 @@ +namespace Microsoft.Extensions.AI +{ + public static class SentryAIExtensions + { + public static Microsoft.Extensions.AI.IChatClient AddSentry(this Microsoft.Extensions.AI.IChatClient client, System.Action? configure = null) { } + public static Microsoft.Extensions.AI.ChatOptions AddSentryToolInstrumentation(this Microsoft.Extensions.AI.ChatOptions options) { } + } +} +namespace Sentry.Extensions.AI +{ + public class SentryAIOptions + { + public SentryAIOptions() { } + public Sentry.Extensions.AI.SentryAIOptions.SentryAIExperimentalOptions Experimental { get; set; } + public sealed class SentryAIExperimentalOptions + { + public SentryAIExperimentalOptions() { } + public string AgentName { get; set; } + public bool RecordInputs { get; set; } + public bool RecordOutputs { get; set; } + } + } +} \ No newline at end of file diff --git a/test/Sentry.Extensions.AI.Tests/Sentry.Extensions.AI.Tests.csproj b/test/Sentry.Extensions.AI.Tests/Sentry.Extensions.AI.Tests.csproj index 8bb05ffc4c..c65ae1bee2 100644 --- a/test/Sentry.Extensions.AI.Tests/Sentry.Extensions.AI.Tests.csproj +++ b/test/Sentry.Extensions.AI.Tests/Sentry.Extensions.AI.Tests.csproj @@ -2,6 +2,11 @@ $(CurrentTfms) + $(TargetFrameworks);net48 + + + + enable diff --git a/test/Sentry.Extensions.AI.Tests/SentryAIActivityListenerTests.cs b/test/Sentry.Extensions.AI.Tests/SentryAIActivityListenerTests.cs index 9b94974df6..e0bc2060a9 100644 --- a/test/Sentry.Extensions.AI.Tests/SentryAIActivityListenerTests.cs +++ b/test/Sentry.Extensions.AI.Tests/SentryAIActivityListenerTests.cs @@ -1,5 +1,3 @@ -#nullable enable - namespace Sentry.Extensions.AI.Tests; public class SentryAIActivityListenerTests diff --git a/test/Sentry.Extensions.AI.Tests/SentryAIExtensionsTests.cs b/test/Sentry.Extensions.AI.Tests/SentryAIExtensionsTests.cs index bd0f25856c..8d03e89d03 100644 --- a/test/Sentry.Extensions.AI.Tests/SentryAIExtensionsTests.cs +++ b/test/Sentry.Extensions.AI.Tests/SentryAIExtensionsTests.cs @@ -1,4 +1,3 @@ -#nullable enable using Microsoft.Extensions.AI; namespace Sentry.Extensions.AI.Tests; diff --git a/test/Sentry.Extensions.AI.Tests/SentryAIOptionsTests.cs b/test/Sentry.Extensions.AI.Tests/SentryAIOptionsTests.cs index 54d013913f..22df50bd05 100644 --- a/test/Sentry.Extensions.AI.Tests/SentryAIOptionsTests.cs +++ b/test/Sentry.Extensions.AI.Tests/SentryAIOptionsTests.cs @@ -1,5 +1,3 @@ -#nullable enable - namespace Sentry.Extensions.AI.Tests; public class SentryAIOptionsTests @@ -7,12 +5,13 @@ public class SentryAIOptionsTests [Fact] public void Constructor_SetsDefaultValues() { - // Act + // Arrange & Act var options = new SentryAIOptions(); // Assert Assert.True(options.Experimental.RecordInputs); Assert.True(options.Experimental.RecordOutputs); + Assert.Equal("Agent", options.Experimental.AgentName); } [Fact] diff --git a/test/Sentry.Extensions.AI.Tests/SentryAISpanEnricherTests.cs b/test/Sentry.Extensions.AI.Tests/SentryAISpanEnricherTests.cs index 5cd12bc00d..4511a63485 100644 --- a/test/Sentry.Extensions.AI.Tests/SentryAISpanEnricherTests.cs +++ b/test/Sentry.Extensions.AI.Tests/SentryAISpanEnricherTests.cs @@ -1,15 +1,14 @@ -#nullable enable using Microsoft.Extensions.AI; namespace Sentry.Extensions.AI.Tests; -public class SentryAISpanEnricherTests +public class SentryAISpanEnricherTests : IDisposable { private class Fixture { private SentryOptions Options { get; } public ISentryClient Client { get; } - public Hub Hub { get; set; } + public Hub Hub { get; } public Fixture() { @@ -26,6 +25,11 @@ public Fixture() private readonly Fixture _fixture = new(); + public void Dispose() + { + _fixture.Hub.Dispose(); + } + private static ChatMessage[] TestMessages() { var initialMessage = new ChatMessage(ChatRole.User, "Hello"); diff --git a/test/Sentry.Extensions.AI.Tests/SentryChatClientTests.cs b/test/Sentry.Extensions.AI.Tests/SentryChatClientTests.cs index b4330b6a9c..dd24358c66 100644 --- a/test/Sentry.Extensions.AI.Tests/SentryChatClientTests.cs +++ b/test/Sentry.Extensions.AI.Tests/SentryChatClientTests.cs @@ -1,16 +1,15 @@ -#nullable enable using Microsoft.Extensions.AI; namespace Sentry.Extensions.AI.Tests; -public class SentryChatClientTests +public class SentryChatClientTests : IDisposable { private class Fixture { private SentryOptions Options { get; } public IHub Hub { get; } public ActivitySource Source { get; } = SentryAIActivitySource.CreateSource(); - public IChatClient InnerClient = Substitute.For(); + public IChatClient InnerClient { get; } = Substitute.For(); public Fixture() { @@ -20,22 +19,25 @@ public Fixture() TracesSampleRate = 1.0, }; - SentrySdk.Init(Options); - Hub = SentrySdk.CurrentHub; + Hub = new Hub(Options); } - public SentryChatClient GetSut() => new SentryChatClient(Source, InnerClient); + public SentryChatClient GetSut() => new SentryChatClient(Source, Hub, InnerClient); } private readonly Fixture _fixture = new(); + public void Dispose() + { + ((Hub)_fixture.Hub).Dispose(); + } + [Fact] public async Task CompleteAsync_CallsInnerClient_AndSetsData() { // Arrange var transaction = _fixture.Hub.StartTransaction("test-nonstreaming", "test"); _fixture.Hub.ConfigureScope(scope => scope.Transaction = transaction); - SentrySdk.ConfigureScope(scope => scope.Transaction = transaction); var message = new ChatMessage(ChatRole.Assistant, "ok"); var chatResponse = new ChatResponse(message); @@ -73,9 +75,14 @@ public async Task CompleteAsync_HandlesErrors_AndFinishesSpanWithException() // Arrange var transaction = _fixture.Hub.StartTransaction("test-nonstreaming", "test"); _fixture.Hub.ConfigureScope(scope => scope.Transaction = transaction); - SentrySdk.ConfigureScope(scope => scope.Transaction = transaction); - + // Simulate FunctionInvokingChatClient's Activity + var listener = SentryAIActivityListener.CreateListener(_fixture.Hub); + var activity = _fixture.Source.StartActivity(SentryAIConstants.FICCActivityNames[0]); var sentryChatClient = _fixture.GetSut(); + var chatClientOption = new ChatOptions + { + Tools = [] + }; var expectedException = new InvalidOperationException("Streaming failed"); _fixture.InnerClient.GetResponseAsync(Arg.Any>(), Arg.Any(), Arg.Any()) @@ -83,7 +90,9 @@ public async Task CompleteAsync_HandlesErrors_AndFinishesSpanWithException() // Act var res = await Assert.ThrowsAsync(async () => - await sentryChatClient.GetResponseAsync([new ChatMessage(ChatRole.User, "hi")])); + await sentryChatClient.GetResponseAsync([new ChatMessage(ChatRole.User, "hi")], chatClientOption)); + activity?.Stop(); + listener.Dispose(); // Assert Assert.Equal(expectedException.Message, res.Message); @@ -105,10 +114,9 @@ public async Task CompleteAsync_HandlesErrors_AndFinishesSpanWithException() [Fact] public async Task CompleteStreamingAsync_CallsInnerClient_AndSetsSpanData() { - // Arrange - Use Fixture Hub to start transaction + // Arrange var transaction = _fixture.Hub.StartTransaction("test-streaming", "test"); _fixture.Hub.ConfigureScope(scope => scope.Transaction = transaction); - SentrySdk.ConfigureScope(scope => scope.Transaction = transaction); _fixture.InnerClient.GetStreamingResponseAsync(Arg.Any>(), Arg.Any(), Arg.Any()) @@ -151,25 +159,28 @@ public async Task CompleteStreamingAsync_HandlesErrors_AndFinishesSpanWithExcept // Arrange var transaction = _fixture.Hub.StartTransaction("test-streaming-error", "test"); _fixture.Hub.ConfigureScope(scope => scope.Transaction = transaction); - SentrySdk.ConfigureScope(scope => scope.Transaction = transaction); var expectedException = new InvalidOperationException("Streaming failed"); _fixture.InnerClient.GetStreamingResponseAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(CreateFailingStreamingUpdatesAsync(expectedException)); var sentryChatClient = _fixture.GetSut(); + var results = new List(); // Act var actualException = await Assert.ThrowsAsync(async () => { await foreach (var update in sentryChatClient.GetStreamingResponseAsync([new ChatMessage(ChatRole.User, "hi")])) { - // Should not reach here due to exception + results.Add(update); } + throw new UnreachableException("Should not reach here due to exception."); }); // Assert Assert.Equal(expectedException.Message, actualException.Message); + var result = Assert.Single(results); + Assert.Equal("Hello", result.Text); var chatSpan = transaction.Spans.FirstOrDefault(s => s.Operation == SentryAIConstants.SpanAttributes.ChatOperation); var agentSpan = transaction.Spans.FirstOrDefault(s => s.Operation == SentryAIConstants.SpanAttributes.InvokeAgentOperation); diff --git a/test/Sentry.Extensions.AI.Tests/SentryInstrumentedFunctionTests.cs b/test/Sentry.Extensions.AI.Tests/SentryInstrumentedFunctionTests.cs index f9ca323dd0..d64964d547 100644 --- a/test/Sentry.Extensions.AI.Tests/SentryInstrumentedFunctionTests.cs +++ b/test/Sentry.Extensions.AI.Tests/SentryInstrumentedFunctionTests.cs @@ -1,4 +1,3 @@ -#nullable enable using Microsoft.Extensions.AI; namespace Sentry.Extensions.AI.Tests; @@ -31,14 +30,8 @@ public async Task InvokeCoreAsync_WithValidFunction_ReturnsResult() // Assert // AIFunctionFactory returns JsonElement, so we need to check the actual content Assert.NotNull(result); - if (result is JsonElement jsonElement) - { - Assert.Equal("test result", jsonElement.GetString()); - } - else - { - Assert.Equal("test result", result); - } + var jsonElement = Assert.IsType(result); + Assert.Equal("test result", jsonElement.GetString()); Assert.Equal("TestFunction", sentryFunction.Name); Assert.Equal("Test function description", sentryFunction.Description); @@ -59,14 +52,9 @@ public async Task InvokeCoreAsync_WithNullResult_ReturnsNull() var result = await sentryFunction.InvokeAsync(arguments); // Assert - if (result is JsonElement jsonElement) - { - Assert.Equal(JsonValueKind.Null, jsonElement.ValueKind); - } - else - { - Assert.Null(result); - } + Assert.NotNull(result); + var jsonElement = Assert.IsType(result); + Assert.Equal(JsonValueKind.Null, jsonElement.ValueKind); _fixture.Hub.Received(1).StartTransaction( Arg.Any(), @@ -87,9 +75,9 @@ public async Task InvokeCoreAsync_WithJsonNullResult_ReturnsJsonElement() // Assert Assert.NotNull(result); - Assert.IsType(result); - var jsonResult = (JsonElement)result; + var jsonResult = Assert.IsType(result); Assert.Equal(JsonValueKind.Null, jsonResult.ValueKind); + _fixture.Hub.Received(1).StartTransaction( Arg.Any(), Arg.Any>()); @@ -109,9 +97,9 @@ public async Task InvokeCoreAsync_WithJsonElementResult_CallsToStringForSpanOutp // Assert Assert.NotNull(result); - Assert.IsType(result); - var jsonResult = (JsonElement)result; + var jsonResult = Assert.IsType(result); Assert.Equal("test output", jsonResult.GetString()); + _fixture.Hub.Received(1).StartTransaction( Arg.Any(), Arg.Any>()); @@ -135,18 +123,11 @@ public async Task InvokeCoreAsync_WithComplexResult_ReturnsObject() // Assert Assert.NotNull(result); - if (result is JsonElement jsonElement) - { - // When AIFunction serializes objects, they become JsonElements - var message = jsonElement.GetProperty("message").GetString(); - var count = jsonElement.GetProperty("count").GetInt32(); - Assert.Equal("test", message); - Assert.Equal(42, count); - } - else - { - Assert.Equal(resultObject, result); - } + var jsonElement = Assert.IsType(result); + var message = jsonElement.GetProperty("message").GetString(); + var count = jsonElement.GetProperty("count").GetInt32(); + Assert.Equal("test", message); + Assert.Equal(42, count); _fixture.Hub.Received(1).StartTransaction( Arg.Any(), @@ -191,6 +172,7 @@ public async Task InvokeCoreAsync_WithCancellation_PropagatesCancellation() // Act & Assert await Assert.ThrowsAsync(async () => await sentryFunction.InvokeAsync(arguments, cts.Token)); + _fixture.Hub.Received(1).StartTransaction( Arg.Any(), Arg.Any>()); @@ -219,6 +201,7 @@ public async Task InvokeCoreAsync_WithParameters_PassesParametersCorrectly() // Assert Assert.NotNull(receivedArguments); Assert.Equal("value1", receivedArguments["param1"]); + _fixture.Hub.Received(1).StartTransaction( Arg.Any(), Arg.Any>());