ref(AI): Microsoft.Extensions.AI Agent Monitoring

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Features
+
+- Added a new SDK `Sentry.Extensions.AI` which allows LLM usage instrumentation via `Microsoft.Extensions.AI` ([#4657](https://github.com/getsentry/sentry-dotnet/pull/4657))
+
 ### Fixes
 
 - Captured [Http Client Errors](https://docs.sentry.io/platforms/dotnet/guides/aspnet/configuration/http-client-errors/) on .NET 5+ now include a full stack trace in order to improve Issue grouping ([#4724](https://github.com/getsentry/sentry-dotnet/pull/4724))
@@ -47,7 +51,6 @@
   - The SDK now makes use of the new SessionEndStatus `Unhandled` when capturing an unhandled but non-terminal exception, i.e. through the UnobservedTaskExceptionIntegration ([#4633](https://github.com/getsentry/sentry-dotnet/pull/4633), [#4653](https://github.com/getsentry/sentry-dotnet/pull/4653))
 - Implemented instance isolation so that multiple instances of the Sentry SDK can be instantiated inside the same process when using the Caching Transport ([#4498](https://github.com/getsentry/sentry-dotnet/pull/4498))
 - Extended the App context by `app_memory` that can hold the amount of memory used by the application in bytes. ([#4707](https://github.com/getsentry/sentry-dotnet/pull/4707))
-- Added a new SDK `Sentry.Extensions.AI` which allows LLM usage instrumentation via `Microsoft.Extensions.AI` ([#4657](https://github.com/getsentry/sentry-dotnet/pull/4657))
 
 ### Fixes
 

samples/Sentry.Samples.ME.AI.AspNetCore/Program.cs

@@ -1,4 +1,3 @@
-#nullable enable
 using Microsoft.Extensions.AI;
 
 var builder = WebApplication.CreateBuilder(args);

samples/Sentry.Samples.ME.AI.AspNetCore/Sentry.Samples.ME.AI.AspNetCore.csproj

@@ -1,7 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>

samples/Sentry.Samples.ME.AI.Console/Program.cs

@@ -96,11 +96,8 @@
     "Please help me with the following tasks: 1) Find Alice's age, 2) Get weather in New York, and 3) Calculate a complex result for number 15. Please use the appropriate tools for each task.",
     options);
 
-logger.LogInformation("Response: {ResponseText}", response.Messages?.FirstOrDefault()?.Text ?? "No response");
+logger.LogInformation("Response: {ResponseText}", response.Messages.FirstOrDefault()?.Text ?? "No response");
 
 logger.LogInformation("Microsoft.Extensions.AI sample completed! Check your Sentry dashboard for the trace data.");
 
 transaction.Finish();
-
-// Flush Sentry to ensure all transactions are sent before the app exits
-await SentrySdk.FlushAsync(TimeSpan.FromSeconds(2));

samples/Sentry.Samples.ME.AI.Console/Sentry.Samples.ME.AI.Console.csproj

@@ -2,7 +2,8 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net9.0</TargetFramework>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>

src/Sentry.Extensions.AI/Sentry.Extensions.AI.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <TargetFrameworks>$(CurrentTfms);netstandard2.0</TargetFrameworks>
     <PackageTags>$(PackageTags);Microsoft.Extensions.AI;AI;LLM</PackageTags>
-    <Description>Microsoft.Extensions.AI integration for Sentry - captures AI Agent telemetry spans per Sentry AI Agents module.</Description>
+    <Description>Official Microsoft.Extensions.AI integration for Sentry - Open-source error tracking that helps developers monitor and fix crashes in real time.</Description>
   </PropertyGroup>
 
   <ItemGroup>

src/Sentry.Extensions.AI/SentryChatClient.cs

@@ -5,19 +5,20 @@ namespace Sentry.Extensions.AI;
 internal sealed class SentryChatClient : DelegatingChatClient
 {
     private readonly ActivitySource _activitySource;
-    private readonly HubAdapter _hub = HubAdapter.Instance;
+    private readonly IHub _hub;
     private readonly SentryAIOptions _sentryAIOptions;
 
-    public SentryChatClient(IChatClient client, Action<SentryAIOptions>? configure = null) : this(null, client, configure)
+    public SentryChatClient(IChatClient client, Action<SentryAIOptions>? configure = null) : this(null, null, client, configure)
     {
     }
 
     /// <summary>
-    /// Internal ovverride for testing
+    /// Internal overload for testing
     /// </summary>
-    internal SentryChatClient(ActivitySource? activitySource, IChatClient client, Action<SentryAIOptions>? configure = null) : base(client)
+    internal SentryChatClient(ActivitySource? activitySource, IHub? hub, IChatClient client, Action<SentryAIOptions>? configure = null) : base(client)
     {
         _activitySource = activitySource ?? SentryAIActivitySource.Instance;
+        _hub = hub ?? HubAdapter.Instance;
         _sentryAIOptions = new SentryAIOptions();
         configure?.Invoke(_sentryAIOptions);
     }

src/Sentry.Extensions.AI/SentryInstrumentedFunction.cs

@@ -6,6 +6,7 @@ internal sealed class SentryInstrumentedFunction(AIFunction innerFunction, IHub?
     : DelegatingAIFunction(innerFunction)
 {
     private readonly IHub _hub = hub ?? HubAdapter.Instance;
+
     protected override async ValueTask<object?> InvokeCoreAsync(
         AIFunctionArguments arguments,
         CancellationToken cancellationToken)

test/Sentry.Extensions.AI.Tests/ApiApprovalTests.Run.Net4_8.verified.txt

@@ -0,0 +1,23 @@
+namespace Microsoft.Extensions.AI
+{
+    public static class SentryAIExtensions
+    {
+        public static Microsoft.Extensions.AI.IChatClient AddSentry(this Microsoft.Extensions.AI.IChatClient client, System.Action<Sentry.Extensions.AI.SentryAIOptions>? configure = null) { }
+        public static Microsoft.Extensions.AI.ChatOptions AddSentryToolInstrumentation(this Microsoft.Extensions.AI.ChatOptions options) { }
+    }
+}
+namespace Sentry.Extensions.AI
+{
+    public class SentryAIOptions
+    {
+        public SentryAIOptions() { }
+        public Sentry.Extensions.AI.SentryAIOptions.SentryAIExperimentalOptions Experimental { get; set; }
+        public sealed class SentryAIExperimentalOptions
+        {
+            public SentryAIExperimentalOptions() { }
+            public string AgentName { get; set; }
+            public bool RecordInputs { get; set; }
+            public bool RecordOutputs { get; set; }
+        }
+    }
+}

test/Sentry.Extensions.AI.Tests/Sentry.Extensions.AI.Tests.csproj

@@ -2,6 +2,11 @@
 
   <PropertyGroup>
     <TargetFrameworks>$(CurrentTfms)</TargetFrameworks>
+    <TargetFrameworks Condition="$([MSBuild]::IsOSPlatform('Windows'))">$(TargetFrameworks);net48</TargetFrameworks>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>

test/Sentry.Extensions.AI.Tests/SentryAIActivityListenerTests.cs

@@ -1,5 +1,3 @@
-#nullable enable
-
 namespace Sentry.Extensions.AI.Tests;
 
 public class SentryAIActivityListenerTests

test/Sentry.Extensions.AI.Tests/SentryAIExtensionsTests.cs

@@ -1,4 +1,3 @@
-#nullable enable
 using Microsoft.Extensions.AI;
 
 namespace Sentry.Extensions.AI.Tests;

test/Sentry.Extensions.AI.Tests/SentryAIOptionsTests.cs

@@ -1,18 +1,17 @@
-#nullable enable
-
 namespace Sentry.Extensions.AI.Tests;
 
 public class SentryAIOptionsTests
 {
     [Fact]
     public void Constructor_SetsDefaultValues()
     {
-        // Act
+        // Arrange & Act
         var options = new SentryAIOptions();
 
         // Assert
         Assert.True(options.Experimental.RecordInputs);
         Assert.True(options.Experimental.RecordOutputs);
+        Assert.Equal("Agent", options.Experimental.AgentName);
     }
 
     [Fact]

test/Sentry.Extensions.AI.Tests/SentryAISpanEnricherTests.cs

@@ -1,15 +1,14 @@
-#nullable enable
 using Microsoft.Extensions.AI;
 
 namespace Sentry.Extensions.AI.Tests;
 
-public class SentryAISpanEnricherTests
+public class SentryAISpanEnricherTests : IDisposable
 {
     private class Fixture
     {
         private SentryOptions Options { get; }
         public ISentryClient Client { get; }
-        public Hub Hub { get; set; }
+        public Hub Hub { get; }
 
         public Fixture()
         {
@@ -26,6 +25,11 @@ public Fixture()
 
     private readonly Fixture _fixture = new();
 
+    public void Dispose()
+    {
+        _fixture.Hub.Dispose();
+    }
+
     private static ChatMessage[] TestMessages()
     {
         var initialMessage = new ChatMessage(ChatRole.User, "Hello");

test/Sentry.Extensions.AI.Tests/SentryChatClientTests.cs

@@ -1,16 +1,15 @@
-#nullable enable
 using Microsoft.Extensions.AI;
 
 namespace Sentry.Extensions.AI.Tests;
 
-public class SentryChatClientTests
+public class SentryChatClientTests : IDisposable
 {
     private class Fixture
     {
         private SentryOptions Options { get; }
         public IHub Hub { get; }
         public ActivitySource Source { get; } = SentryAIActivitySource.CreateSource();
-        public IChatClient InnerClient = Substitute.For<IChatClient>();
+        public IChatClient InnerClient { get; } = Substitute.For<IChatClient>();
 
         public Fixture()
         {
@@ -20,22 +19,25 @@ public Fixture()
                 TracesSampleRate = 1.0,
             };
 
-            SentrySdk.Init(Options);
-            Hub = SentrySdk.CurrentHub;
+            Hub = new Hub(Options);
         }
 
-        public SentryChatClient GetSut() => new SentryChatClient(Source, InnerClient);
+        public SentryChatClient GetSut() => new SentryChatClient(Source, Hub, InnerClient);
     }
 
     private readonly Fixture _fixture = new();
 
+    public void Dispose()
+    {
+        ((Hub)_fixture.Hub).Dispose();
+    }
+
     [Fact]
     public async Task CompleteAsync_CallsInnerClient_AndSetsData()
     {
         // Arrange
         var transaction = _fixture.Hub.StartTransaction("test-nonstreaming", "test");
         _fixture.Hub.ConfigureScope(scope => scope.Transaction = transaction);
-        SentrySdk.ConfigureScope(scope => scope.Transaction = transaction);
 
         var message = new ChatMessage(ChatRole.Assistant, "ok");
         var chatResponse = new ChatResponse(message);
@@ -73,17 +75,24 @@ public async Task CompleteAsync_HandlesErrors_AndFinishesSpanWithException()
         // Arrange
         var transaction = _fixture.Hub.StartTransaction("test-nonstreaming", "test");
         _fixture.Hub.ConfigureScope(scope => scope.Transaction = transaction);
-        SentrySdk.ConfigureScope(scope => scope.Transaction = transaction);
-
+        // Simulate FunctionInvokingChatClient's Activity
+        var listener = SentryAIActivityListener.CreateListener(_fixture.Hub);
+        var activity = _fixture.Source.StartActivity(SentryAIConstants.FICCActivityNames[0]);
         var sentryChatClient = _fixture.GetSut();
+        var chatClientOption = new ChatOptions
+        {
+            Tools = []
+        };
         var expectedException = new InvalidOperationException("Streaming failed");
 
         _fixture.InnerClient.GetResponseAsync(Arg.Any<IList<ChatMessage>>(), Arg.Any<ChatOptions>(), Arg.Any<CancellationToken>())
             .Throws(expectedException);
 
         // Act
         var res = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
-            await sentryChatClient.GetResponseAsync([new ChatMessage(ChatRole.User, "hi")]));
+            await sentryChatClient.GetResponseAsync([new ChatMessage(ChatRole.User, "hi")], chatClientOption));
+        activity?.Stop();
+        listener.Dispose();
 
         // Assert
         Assert.Equal(expectedException.Message, res.Message);
@@ -105,10 +114,9 @@ public async Task CompleteAsync_HandlesErrors_AndFinishesSpanWithException()
     [Fact]
     public async Task CompleteStreamingAsync_CallsInnerClient_AndSetsSpanData()
     {
-        // Arrange - Use Fixture Hub to start transaction
+        // Arrange
         var transaction = _fixture.Hub.StartTransaction("test-streaming", "test");
         _fixture.Hub.ConfigureScope(scope => scope.Transaction = transaction);
-        SentrySdk.ConfigureScope(scope => scope.Transaction = transaction);
 
         _fixture.InnerClient.GetStreamingResponseAsync(Arg.Any<IList<ChatMessage>>(), Arg.Any<ChatOptions>(),
                 Arg.Any<CancellationToken>())
@@ -151,25 +159,28 @@ public async Task CompleteStreamingAsync_HandlesErrors_AndFinishesSpanWithExcept
         // Arrange
         var transaction = _fixture.Hub.StartTransaction("test-streaming-error", "test");
         _fixture.Hub.ConfigureScope(scope => scope.Transaction = transaction);
-        SentrySdk.ConfigureScope(scope => scope.Transaction = transaction);
 
         var expectedException = new InvalidOperationException("Streaming failed");
         _fixture.InnerClient.GetStreamingResponseAsync(Arg.Any<IList<ChatMessage>>(), Arg.Any<ChatOptions>(),
                 Arg.Any<CancellationToken>())
             .Returns(CreateFailingStreamingUpdatesAsync(expectedException));
         var sentryChatClient = _fixture.GetSut();
+        var results = new List<ChatResponseUpdate>();
 
         // Act
         var actualException = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
         {
             await foreach (var update in sentryChatClient.GetStreamingResponseAsync([new ChatMessage(ChatRole.User, "hi")]))
             {
-                // Should not reach here due to exception
+                results.Add(update);
             }
+            throw new UnreachableException("Should not reach here due to exception.");
         });
 
         // Assert
         Assert.Equal(expectedException.Message, actualException.Message);
+        var result = Assert.Single(results);
+        Assert.Equal("Hello", result.Text);
 
         var chatSpan = transaction.Spans.FirstOrDefault(s => s.Operation == SentryAIConstants.SpanAttributes.ChatOperation);
         var agentSpan = transaction.Spans.FirstOrDefault(s => s.Operation == SentryAIConstants.SpanAttributes.InvokeAgentOperation);