-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net: Add Bedrock Agent tests (#10618)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> PR for Bedrock Agent in .Net SK has been merged: #10443. This PR adds tests to the integration. ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> Add unit tests and integration tests. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄
- Loading branch information
1 parent
2482cb9
commit 2794352
Showing
11 changed files
with
1,274 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
289 changes: 289 additions & 0 deletions
289
dotnet/src/Agents/UnitTests/Bedrock/BedrockAgentChannelTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,289 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Amazon.BedrockAgent; | ||
| using Amazon.BedrockAgentRuntime; | ||
| using Amazon.BedrockAgentRuntime.Model; | ||
| using Microsoft.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Agents.Bedrock; | ||
| using Microsoft.SemanticKernel.ChatCompletion; | ||
| using Moq; | ||
| using Xunit; | ||
|
|
||
| namespace SemanticKernel.Agents.UnitTests.Bedrock; | ||
|
|
||
| /// <summary> | ||
| /// Unit testing of <see cref="BedrockAgentChannel"/>. | ||
| /// </summary> | ||
| public class BedrockAgentChannelTests | ||
| { | ||
| private readonly Amazon.BedrockAgent.Model.Agent _agentModel = new() | ||
| { | ||
| AgentId = "1234567890", | ||
| AgentName = "testName", | ||
| Description = "test description", | ||
| Instruction = "Instruction must have at least 40 characters", | ||
| }; | ||
|
|
||
| /// <summary> | ||
| /// Verify the simple scenario of receiving messages in a <see cref="BedrockAgentChannel"/>. | ||
| /// </summary> | ||
| [Fact] | ||
| public async Task VerifyReceiveAsync() | ||
| { | ||
| // Arrange | ||
| BedrockAgentChannel channel = new(); | ||
| List<ChatMessageContent> history = this.CreateNormalHistory(); | ||
|
|
||
| // Act | ||
| await channel.ReceiveAsync(history); | ||
|
|
||
| // Assert | ||
| Assert.Equal(2, await channel.GetHistoryAsync().CountAsync()); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Verify the <see cref="BedrockAgentChannel"/> skips messages with empty content. | ||
| /// </summary> | ||
| [Fact] | ||
| public async Task VerifyReceiveWithEmptyContentAsync() | ||
| { | ||
| // Arrange | ||
| BedrockAgentChannel channel = new(); | ||
| List<ChatMessageContent> history = [ | ||
| new ChatMessageContent() | ||
| { | ||
| Role = AuthorRole.User, | ||
| }, | ||
| ]; | ||
|
|
||
| // Act | ||
| await channel.ReceiveAsync(history); | ||
|
|
||
| // Assert | ||
| Assert.Empty(await channel.GetHistoryAsync().ToArrayAsync()); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Verify the channel inserts placeholders when the message sequence is incorrect. | ||
| /// </summary> | ||
| [Fact] | ||
| public async Task VerifyReceiveWithIncorrectSequenceAsync() | ||
| { | ||
| // Arrange | ||
| BedrockAgentChannel channel = new(); | ||
| List<ChatMessageContent> history = this.CreateIncorrectSequenceHistory(); | ||
|
|
||
| // Act | ||
| await channel.ReceiveAsync(history); | ||
|
|
||
| // Assert that a user message is inserted between the two agent messages. | ||
| // Note that `GetHistoryAsync` returns the history in a reversed order. | ||
| Assert.Equal(6, await channel.GetHistoryAsync().CountAsync()); | ||
| Assert.Equal(AuthorRole.User, (await channel.GetHistoryAsync().ToArrayAsync())[3].Role); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Verify the channel empties the history when reset. | ||
| /// </summary> | ||
| [Fact] | ||
| public async Task VerifyResetAsync() | ||
| { | ||
| // Arrange | ||
| BedrockAgentChannel channel = new(); | ||
| List<ChatMessageContent> history = this.CreateNormalHistory(); | ||
|
|
||
| // Act | ||
| await channel.ReceiveAsync(history); | ||
|
|
||
| // Assert | ||
| Assert.NotEmpty(await channel.GetHistoryAsync().ToArrayAsync()); | ||
|
|
||
| // Act | ||
| await channel.ResetAsync(); | ||
|
|
||
| // Assert | ||
| Assert.Empty(await channel.GetHistoryAsync().ToArrayAsync()); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Verify the channel correctly prepares the history for invocation. | ||
| /// </summary> | ||
| [Fact] | ||
| public async Task VerifyInvokeAsync() | ||
| { | ||
| // Arrange | ||
| var (mockClient, mockRuntimeClient) = this.CreateMockClients(); | ||
| BedrockAgent agent = new(this._agentModel, mockClient.Object, mockRuntimeClient.Object); | ||
|
|
||
| BedrockAgentChannel channel = new(); | ||
| List<ChatMessageContent> history = this.CreateIncorrectSequenceHistory(); | ||
|
|
||
| // Act | ||
| async Task InvokeAgent() | ||
| { | ||
| await channel.ReceiveAsync(history); | ||
| await foreach (var _ in channel.InvokeAsync(agent)) | ||
| { | ||
| continue; | ||
| } | ||
| } | ||
|
|
||
| // Assert | ||
| await Assert.ThrowsAsync<HttpOperationException>(() => InvokeAgent()); | ||
| mockRuntimeClient.Verify(x => x.InvokeAgentAsync( | ||
| It.Is<InvokeAgentRequest>(r => | ||
| r.AgentAliasId == BedrockAgent.WorkingDraftAgentAlias | ||
| && r.AgentId == this._agentModel.AgentId | ||
| && r.InputText == "[SILENCE]" // Inserted by `EnsureLastMessageIsUser`. | ||
| && r.SessionState.ConversationHistory.Messages.Count == 6 // There is also a user message inserted between the two agent messages. | ||
| ), | ||
| It.IsAny<CancellationToken>() | ||
| ), Times.Once); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Verify the channel returns an empty stream when invoking with an empty history. | ||
| /// </summary> | ||
| [Fact] | ||
| public async Task VerifyInvokeWithEmptyHistoryAsync() | ||
| { | ||
| // Arrange | ||
| var (mockClient, mockRuntimeClient) = this.CreateMockClients(); | ||
| BedrockAgent agent = new(this._agentModel, mockClient.Object, mockRuntimeClient.Object); | ||
|
|
||
| BedrockAgentChannel channel = new(); | ||
|
|
||
| // Act | ||
| List<ChatMessageContent> history = []; | ||
| await foreach ((bool _, ChatMessageContent Message) in channel.InvokeAsync(agent)) | ||
| { | ||
| history.Add(Message); | ||
| } | ||
|
|
||
| // Assert | ||
| Assert.Empty(history); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Verify the channel correctly prepares the history for streaming invocation. | ||
| /// </summary> | ||
| [Fact] | ||
| public async Task VerifyInvokeStreamAsync() | ||
| { | ||
| // Arrange | ||
| var (mockClient, mockRuntimeClient) = this.CreateMockClients(); | ||
| BedrockAgent agent = new(this._agentModel, mockClient.Object, mockRuntimeClient.Object); | ||
|
|
||
| BedrockAgentChannel channel = new(); | ||
| List<ChatMessageContent> history = this.CreateIncorrectSequenceHistory(); | ||
|
|
||
| // Act | ||
| async Task InvokeAgent() | ||
| { | ||
| await channel.ReceiveAsync(history); | ||
| await foreach (var _ in channel.InvokeStreamingAsync(agent, [])) | ||
| { | ||
| continue; | ||
| } | ||
| } | ||
|
|
||
| // Assert | ||
| await Assert.ThrowsAsync<HttpOperationException>(() => InvokeAgent()); | ||
| mockRuntimeClient.Verify(x => x.InvokeAgentAsync( | ||
| It.Is<InvokeAgentRequest>(r => | ||
| r.AgentAliasId == BedrockAgent.WorkingDraftAgentAlias | ||
| && r.AgentId == this._agentModel.AgentId | ||
| && r.InputText == "[SILENCE]" // Inserted by `EnsureLastMessageIsUser`. | ||
| && r.SessionState.ConversationHistory.Messages.Count == 6 // There is also a user message inserted between the two agent messages. | ||
| ), | ||
| It.IsAny<CancellationToken>() | ||
| ), Times.Once); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Verify the channel returns an empty stream when invoking with an empty history. | ||
| /// </summary> | ||
| [Fact] | ||
| public async Task VerifyInvokeStreamingWithEmptyHistoryAsync() | ||
| { | ||
| // Arrange | ||
| var (mockClient, mockRuntimeClient) = this.CreateMockClients(); | ||
| BedrockAgent agent = new(this._agentModel, mockClient.Object, mockRuntimeClient.Object); | ||
|
|
||
| BedrockAgentChannel channel = new(); | ||
|
|
||
| // Act | ||
| List<StreamingChatMessageContent> history = []; | ||
| await foreach (var message in channel.InvokeStreamingAsync(agent, [])) | ||
| { | ||
| history.Add(message); | ||
| } | ||
|
|
||
| // Assert | ||
| Assert.Empty(history); | ||
| } | ||
|
|
||
| private List<ChatMessageContent> CreateNormalHistory() | ||
| { | ||
| return | ||
| [ | ||
| new ChatMessageContent(AuthorRole.User, "Hi!"), | ||
| new ChatMessageContent(AuthorRole.Assistant, "Hi, how can I help you?"), | ||
| ]; | ||
| } | ||
|
|
||
| private List<ChatMessageContent> CreateIncorrectSequenceHistory() | ||
| { | ||
| return | ||
| [ | ||
| new ChatMessageContent(AuthorRole.User, "What is a word that starts with 'x'?"), | ||
| new ChatMessageContent(AuthorRole.Assistant, "Xylophone.") | ||
| { | ||
| AuthorName = "Agent 1" | ||
| }, | ||
| new ChatMessageContent(AuthorRole.Assistant, "Xenon.") | ||
| { | ||
| AuthorName = "Agent 2" | ||
| }, | ||
| new ChatMessageContent(AuthorRole.User, "Thanks!"), | ||
| new ChatMessageContent(AuthorRole.Assistant, "Is there anything else you need?") | ||
| { | ||
| AuthorName = "Agent 1" | ||
| }, | ||
| ]; | ||
| } | ||
|
|
||
| private (Mock<AmazonBedrockAgentClient>, Mock<AmazonBedrockAgentRuntimeClient>) CreateMockClients() | ||
| { | ||
| #pragma warning disable Moq1410 // Moq: Set MockBehavior to Strict | ||
| Mock<AmazonBedrockAgentConfig> mockClientConfig = new(); | ||
| Mock<AmazonBedrockAgentRuntimeConfig> mockRuntimeClientConfig = new(); | ||
| mockClientConfig.Setup(x => x.Validate()).Verifiable(); | ||
| mockRuntimeClientConfig.Setup(x => x.Validate()).Verifiable(); | ||
| Mock<AmazonBedrockAgentClient> mockClient = new( | ||
| "fakeAccessId", | ||
| "fakeSecretKey", | ||
| mockClientConfig.Object); | ||
| Mock<AmazonBedrockAgentRuntimeClient> mockRuntimeClient = new( | ||
| "fakeAccessId", | ||
| "fakeSecretKey", | ||
| mockRuntimeClientConfig.Object); | ||
| #pragma warning restore Moq1410 // Moq: Set MockBehavior to Strict | ||
| mockRuntimeClient.Setup(x => x.InvokeAgentAsync( | ||
| It.IsAny<InvokeAgentRequest>(), | ||
| It.IsAny<CancellationToken>()) | ||
| ).ReturnsAsync(new InvokeAgentResponse() | ||
| { | ||
| // It's not important what the response is for this test. | ||
| // And it's difficult to mock the response stream. | ||
| // Tests should expect an exception to be thrown. | ||
| HttpStatusCode = System.Net.HttpStatusCode.NotFound, | ||
| }); | ||
|
|
||
| return (mockClient, mockRuntimeClient); | ||
| } | ||
| } |
Oops, something went wrong.