.Net: Increase code coverage and add threshold to build pipeline (microsoft#4610)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

Resolves: microsoft#4372

Added more tests and increased code coverage for production packages to
meet 80% threshold. Added new steps to PR build pipeline to check code
coverage for production packages and trigger a failure if code coverage
in PR is lower than defined threshold.

Example of code coverage output:

![image](https://github.com/microsoft/semantic-kernel/assets/13853051/2a271dcf-5e87-4006-a3b1-5bcfce829f3a)

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

Author: dmytrostruk

.github/workflows/check-coverage.ps1

@@ -0,0 +1,54 @@
+param (
+    [string]$JsonReportPath,
+    [double]$CoverageThreshold
+)
+
+$jsonContent = Get-Content $JsonReportPath -Raw | ConvertFrom-Json
+$coverageBelowThreshold = $false
+
+function Get-FormattedValue($number) {
+    $formattedNumber = "{0:N1}" -f $number
+    $icon = if ($number -ge $CoverageThreshold) { '✅' } else { '❌' }
+              
+    return "$formattedNumber% $icon"
+}
+
+$lineCoverage = $jsonContent.summary.linecoverage
+$branchCoverage = $jsonContent.summary.branchcoverage
+
+if ($lineCoverage -lt $CoverageThreshold -or $branchCoverage -lt $CoverageThreshold) {
+    $coverageBelowThreshold = $true
+}
+
+$totalTableData = [PSCustomObject]@{
+    'Metric'          = 'Total Coverage'
+    'Line Coverage'   = Get-FormattedValue $lineCoverage
+    'Branch Coverage' = Get-FormattedValue $branchCoverage
+}
+
+$totalTableData | Format-Table -AutoSize
+
+$assemblyTableData = @()
+
+foreach ($assembly in $jsonContent.coverage.assemblies) {
+    $assemblyName = $assembly.name
+    $assemblyLineCoverage = $assembly.coverage
+    $assemblyBranchCoverage = $assembly.branchcoverage
+
+    if ($assemblyLineCoverage -lt $CoverageThreshold -or $assemblyBranchCoverage -lt $CoverageThreshold) {
+        $coverageBelowThreshold = $true
+    }
+
+    $assemblyTableData += [PSCustomObject]@{
+        'Assembly Name' = $assemblyName
+        'Line'          = Get-FormattedValue $assemblyLineCoverage
+        'Branch'        = Get-FormattedValue $assemblyBranchCoverage
+    }
+}
+
+$assemblyTableData | Format-Table -AutoSize
+
+if ($coverageBelowThreshold) {
+    Write-Host "Code coverage is lower than defined threshold: $CoverageThreshold. Stopping the task."
+    exit 1
+}

.github/workflows/dotnet-build-and-test.yml

@@ -12,6 +12,9 @@ on:
   merge_group:
     branches: ["main"]
 
+env:
+  COVERAGE_THRESHOLD: 80
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -82,7 +85,7 @@ jobs:
         run: |
           export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | grep -v -E "(Planners.Core.UnitTests.csproj|Experimental.Orchestration.Flow.UnitTests.csproj|Experimental.Assistants.UnitTests.csproj)" | tr '\n' ' ')
           for project in $UT_PROJECTS; do
-            dotnet test -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx
+            dotnet test -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx --collect:"XPlat Code Coverage" --results-directory:"TestResults/Coverage/"
           done
 
       - name: Run Integration Tests
@@ -109,6 +112,20 @@ jobs:
           Bing__ApiKey: ${{ secrets.BING__APIKEY }}
           OpenAI__ApiKey: ${{ secrets.OPENAI__APIKEY }}
 
+      # Generate test reports and check coverage
+      - name: Generate test reports
+        uses: danielpalme/[email protected]
+        with:
+          reports: "./TestResults/Coverage/**/coverage.cobertura.xml"
+          targetdir: "./TestResults/Reports"
+          reporttypes: "JsonSummary"
+          # Report for production packages only
+          assemblyfilters: "+Microsoft.SemanticKernel.Abstractions;+Microsoft.SemanticKernel.Core;+Microsoft.SemanticKernel.PromptTemplates.Handlebars;+Microsoft.SemanticKernel.Connectors.OpenAI;+Microsoft.SemanticKernel.Yaml;"
+
+      - name: Check coverage
+        shell: pwsh
+        run: .github/workflows/check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD
+
   # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed
   dotnet-build-and-test-check:
     if: always()

dotnet/SK-dotnet.sln

@@ -118,6 +118,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "System", "System", "{3CDE10
 		src\InternalUtilities\src\System\EnvExtensions.cs = src\InternalUtilities\src\System\EnvExtensions.cs
 	EndProjectSection
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Type", "Type", "{E85EA4D0-BB7E-4DFD-882F-A76EB8C0B8FF}"
+	ProjectSection(SolutionItems) = preProject
+		src\InternalUtilities\src\Type\TypeExtensions.cs = src\InternalUtilities\src\Type\TypeExtensions.cs
+	EndProjectSection
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Plugins.Core", "src\Plugins\Plugins.Core\Plugins.Core.csproj", "{0D0C4DAD-E6BC-4504-AE3A-EEA4E35920C1}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NCalcPlugins", "samples\NCalcPlugins\NCalcPlugins.csproj", "{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA}"
@@ -500,6 +505,7 @@ Global
 		{B00AD427-0047-4850-BEF9-BA8237EA9D8B} = {958AD708-F048-4FAF-94ED-D2F2B92748B9}
 		{1C19D805-3573-4477-BF07-40180FCDE1BD} = {958AD708-F048-4FAF-94ED-D2F2B92748B9}
 		{3CDE10B2-AE8F-4FC4-8D55-92D4AD32E144} = {958AD708-F048-4FAF-94ED-D2F2B92748B9}
+		{E85EA4D0-BB7E-4DFD-882F-A76EB8C0B8FF} = {958AD708-F048-4FAF-94ED-D2F2B92748B9}
 		{0D0C4DAD-E6BC-4504-AE3A-EEA4E35920C1} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132}
 		{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA} = {FA3720F1-C99A-49B2-9577-A940257098BF}
 		{C754950A-E16C-4F96-9CC7-9328E361B5AF} = {FA3720F1-C99A-49B2-9577-A940257098BF}

dotnet/code-coverage.ps1

@@ -0,0 +1,60 @@
+# This script is for local use to analyze code coverage in more detail using HTML report.
+
+Param(
+    [switch]$ProdPackagesOnly = $false
+)
+
+# Generate a timestamp for the current date and time
+$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
+
+# Define paths
+$scriptPath = Get-Item -Path $PSScriptRoot
+$coverageOutputPath = Join-Path $scriptPath "TestResults\Coverage\$timestamp"
+$reportOutputPath = Join-Path $scriptPath "TestResults\Reports\$timestamp"
+
+# Create output directories
+New-Item -ItemType Directory -Force -Path $coverageOutputPath
+New-Item -ItemType Directory -Force -Path $reportOutputPath
+
+# Find tests for projects ending with 'UnitTests.csproj'
+$testProjects = Get-ChildItem $scriptPath -Filter "*UnitTests.csproj" -Recurse
+
+foreach ($project in $testProjects) {
+    $testProjectPath = $project.FullName
+    Write-Host "Running tests for project: $($testProjectPath)"
+
+    # Run tests
+    dotnet test $testProjectPath `
+        --collect:"XPlat Code Coverage" `
+        --results-directory:$coverageOutputPath `
+
+}
+
+# Install required tools
+& dotnet tool install -g coverlet.console
+& dotnet tool install -g dotnet-reportgenerator-globaltool
+
+# Generate HTML report
+if ($ProdPackagesOnly) {
+    $assemblies = @(
+        "+Microsoft.SemanticKernel.Abstractions",
+        "+Microsoft.SemanticKernel.Core",
+        "+Microsoft.SemanticKernel.PromptTemplates.Handlebars",
+        "+Microsoft.SemanticKernel.Connectors.OpenAI",
+        "+Microsoft.SemanticKernel.Yaml"
+    )
+
+    $assemblyFilters = $assemblies -join ";"
+
+    # Generate report for production assemblies only
+    & reportgenerator -reports:"$coverageOutputPath/**/coverage.cobertura.xml" -targetdir:$reportOutputPath -reporttypes:Html -assemblyfilters:$assemblyFilters
+}
+else {
+    & reportgenerator -reports:"$coverageOutputPath/**/coverage.cobertura.xml" -targetdir:$reportOutputPath -reporttypes:Html
+}
+
+Write-Host "Code coverage report generated at: $reportOutputPath"
+
+# Open report
+$reportIndexHtml = Join-Path $reportOutputPath "index.html"
+Invoke-Item -Path $reportIndexHtml

dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/AzureOpenAIClientCore.cs

@@ -2,7 +2,6 @@
 
 using System;
 using System.Net.Http;
-using System.Runtime.CompilerServices;
 using Azure;
 using Azure.AI.OpenAI;
 using Azure.Core;
@@ -98,16 +97,4 @@ internal AzureOpenAIClientCore(
 
         this.AddAttribute(DeploymentNameKey, deploymentName);
     }
-
-    /// <summary>
-    /// Logs Azure OpenAI action details.
-    /// </summary>
-    /// <param name="callerMemberName">Caller member name. Populated automatically by runtime.</param>
-    internal void LogActionDetails([CallerMemberName] string? callerMemberName = default)
-    {
-        if (this.Logger.IsEnabled(LogLevel.Information))
-        {
-            this.Logger.LogInformation("Action: {Action}. Azure OpenAI Deployment Name: {DeploymentName}.", callerMemberName, this.DeploymentOrModelName);
-        }
-    }
 }

dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/AzureOpenAIWithDataChatMessageContent.cs

@@ -30,7 +30,8 @@ internal AzureOpenAIWithDataChatMessageContent(ChatWithDataChoice chatChoice, st
         : base(default, string.Empty, modelId, chatChoice, System.Text.Encoding.UTF8, CreateMetadataDictionary(metadata))
     {
         // An assistant message content must be present, otherwise the chat is not valid.
-        var chatMessage = chatChoice.Messages.First(m => string.Equals(m.Role, AuthorRole.Assistant.Label, StringComparison.OrdinalIgnoreCase));
+        var chatMessage = chatChoice.Messages.FirstOrDefault(m => string.Equals(m.Role, AuthorRole.Assistant.Label, StringComparison.OrdinalIgnoreCase)) ??
+            throw new ArgumentException("Chat is not valid. Chat message does not contain any messages with 'assistant' role.");
 
         this.Content = chatMessage.Content;
         this.Role = new AuthorRole(chatMessage.Role);

dotnet/src/Connectors/Connectors.OpenAI/TextEmbedding/AzureOpenAITextEmbeddingGenerationService.cs

@@ -93,7 +93,6 @@ public Task<IList<ReadOnlyMemory<float>>> GenerateEmbeddingsAsync(
         Kernel? kernel = null,
         CancellationToken cancellationToken = default)
     {
-        this._core.LogActionDetails();
         return this._core.GetEmbeddingsAsync(data, kernel, cancellationToken);
     }
 }

dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs

@@ -2,7 +2,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Net.Http;
 using System.Text.Json;
@@ -87,10 +86,7 @@ private async Task<string> GenerateImageAsync(
         string format, Func<TextToImageResponse.Image, string> extractResponse,
         CancellationToken cancellationToken)
     {
-        Debug.Assert(width == height);
-        Debug.Assert(width is 256 or 512 or 1024);
-        Debug.Assert(format is "url" or "b64_json");
-        Debug.Assert(extractResponse is not null);
+        Verify.NotNull(extractResponse);
 
         var requestBody = JsonSerializer.Serialize(new TextToImageRequest
         {

dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj

@@ -27,14 +27,17 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
+    <PackageReference Include="coverlet.collector">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
     <PackageReference Include="System.Numerics.Tensors" />
     <PackageReference Include="System.Text.Json" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
   </ItemGroup>
 
   <ItemGroup>
-    <Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/test/AssertExtensions.cs"
-             Link="%(RecursiveDir)%(Filename)%(Extension)" />
+    <Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/test/AssertExtensions.cs" Link="%(RecursiveDir)%(Filename)%(Extension)" />
   </ItemGroup>
 
   <ItemGroup>
@@ -55,16 +58,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Update="HuggingFace\TestData\completion_test_response.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="HuggingFace\TestData\embeddings_test_response.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="OpenAI\TestData\image_result_test_response.json">
+    <None Update="HuggingFace\TestData\*">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
-    <None Update="OpenAI\TestData\image_generation_test_response.json">
+    <None Update="OpenAI\TestData\*">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
   </ItemGroup>

dotnet/src/Connectors/Connectors.UnitTests/MultipleHttpMessageHandlerStub.cs

@@ -0,0 +1,53 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace SemanticKernel.Connectors.UnitTests;
+
+internal sealed class MultipleHttpMessageHandlerStub : DelegatingHandler
+{
+    private int _callIteration = 0;
+
+    public List<HttpRequestHeaders?> RequestHeaders { get; private set; }
+
+    public List<HttpContentHeaders?> ContentHeaders { get; private set; }
+
+    public List<byte[]?> RequestContents { get; private set; }
+
+    public List<Uri?> RequestUris { get; private set; }
+
+    public List<HttpMethod?> Methods { get; private set; }
+
+    public List<HttpResponseMessage> ResponsesToReturn { get; set; }
+
+    public MultipleHttpMessageHandlerStub()
+    {
+        this.RequestHeaders = [];
+        this.ContentHeaders = [];
+        this.RequestContents = [];
+        this.RequestUris = [];
+        this.Methods = [];
+        this.ResponsesToReturn = [];
+    }
+
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        this._callIteration++;
+
+        this.Methods.Add(request.Method);
+        this.RequestUris.Add(request.RequestUri);
+        this.RequestHeaders.Add(request.Headers);
+        this.ContentHeaders.Add(request.Content?.Headers);
+
+        var content = request.Content == null ? null : await request.Content.ReadAsByteArrayAsync(cancellationToken);
+
+        this.RequestContents.Add(content);
+
+        return await Task.FromResult(this.ResponsesToReturn[this._callIteration - 1]);
+    }
+}