.Net: Mechanism for transforming OpenAPI documents - part2 (#9689)

### Motivation and Context
This is the second PR to add a mechanism to transform OpenAPI documents
before creating a kernel plugin from them.

### Description
This PR:
1. Makes the `OpenApiDocumentParser` public, allowing for the parsing of
OpenAPI documents and accessing the instance of the
`RestApiSpecification` class representing the parsed document, which can
be modified, by the consumer, if needed, before creating a plugin from
it. Currently, it's only possible to modify argument name property of
parameters and server variables.

2. Adds a few kernel extension overload methods to create and import
OpenAPI document represented by the `RestApiSpecification` model class.
This is the final element of the transformation mechanism that receives
the specification model class instance returned by the parser and
transformed by the consumer and creates an SK plugin from it.

3. Adds the `OpenApiDocumentParserOptions` class to represent existing
parser options and allows adding new ones with no breaking changes.

4. Replaces the `operationsToExclude` exclusion list, which is limited
to filtering out operations by operation id, with the
`OperationSelectionPredicate` callback that can filter out operations
based on id, method, path, and description.

5. Removes the **internal** `IOpenApiDocumentParser` unneeded interface
that was not used all that time.

6. Other: XML comments + OpenAPI specification transformation sample

Contributes to the issue:
#4666
The first PR: #9668

dotnet/samples/Concepts/Concepts.csproj

@@ -128,4 +128,9 @@
     <None Remove="Resources\Plugins\EventPlugin\openapiV1.json" />
     <None Remove="Resources\Plugins\EventPlugin\openapiV2.json" />
   </ItemGroup>
+  <ItemGroup>
+    <None Update="Resources\Plugins\ProductsPlugin\openapi.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 </Project>

dotnet/samples/Concepts/Plugins/CreatePluginFromOpenApiSpec_RepairService.cs

@@ -36,7 +36,7 @@ public async Task ShowCreatingRepairServicePluginAsync()
         Console.WriteLine(result.ToString());
 
         // List All Repairs
-        result = await plugin["listRepairs"].InvokeAsync(kernel, arguments);
+        result = await plugin["listRepairs"].InvokeAsync(kernel);
         var repairs = JsonSerializer.Deserialize<Repair[]>(result.ToString());
         Assert.True(repairs?.Length > 0);
         var id = repairs[repairs.Length - 1].Id;

dotnet/samples/Concepts/Plugins/OpenApiPlugin_Customization.cs

@@ -0,0 +1,137 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Plugins.OpenApi;
+
+namespace Plugins;
+
+/// <summary>
+/// These samples show different ways OpenAPI document can be transformed to change its various aspects before creating a plugin out of it.
+/// The transformations can be useful if the original OpenAPI document can't be consumed as is.
+/// </summary>
+public sealed class OpenApiPlugin_Customization : BaseTest
+{
+    private readonly Kernel _kernel;
+    private readonly ITestOutputHelper _output;
+    private readonly HttpClient _httpClient;
+
+    public OpenApiPlugin_Customization(ITestOutputHelper output) : base(output)
+    {
+        IKernelBuilder builder = Kernel.CreateBuilder();
+
+        this._kernel = builder.Build();
+
+        this._output = output;
+
+        void RequestDataHandler(string requestData)
+        {
+            this._output.WriteLine("Request payload");
+            this._output.WriteLine(requestData);
+        }
+
+        // Create HTTP client with a stub handler to log the request data
+        this._httpClient = new(new StubHttpHandler(RequestDataHandler));
+    }
+
+    /// <summary>
+    /// This sample demonstrates how to assign argument names to parameters and variables that have the same name.
+    /// For example, in this sample, there are multiple parameters named 'id' in the 'getProductFromCart' operation.
+    ///   * Region of the API in the server variable.
+    ///   * User ID in the path.
+    ///   * Subscription ID in the query string.
+    ///   * Session ID in the header.
+    /// </summary>
+    [Fact]
+    public async Task HandleOpenApiDocumentHavingTwoParametersWithSameNameButRelatedToDifferentEntitiesAsync()
+    {
+        OpenApiDocumentParser parser = new();
+
+        using StreamReader sr = File.OpenText("Resources/Plugins/ProductsPlugin/openapi.json");
+
+        // Register the custom HTTP client with the stub handler
+        OpenApiFunctionExecutionParameters executionParameters = new() { HttpClient = this._httpClient };
+
+        // Parse the OpenAPI document
+        RestApiSpecification specification = await parser.ParseAsync(sr.BaseStream);
+
+        // Get the 'getProductFromCart' operation
+        RestApiOperation getProductFromCartOperation = specification.Operations.Single(o => o.Id == "getProductFromCart");
+
+        // Set the 'region' argument name to the 'id' server variable that represents the region of the API
+        RestApiServerVariable idServerVariable = getProductFromCartOperation.Servers[0].Variables["id"];
+        idServerVariable.ArgumentName = "region";
+
+        // Set the 'userId' argument name to the 'id' path parameter that represents the user ID
+        RestApiParameter idPathParameter = getProductFromCartOperation.Parameters.Single(p => p.Location == RestApiParameterLocation.Path && p.Name == "id");
+        idPathParameter.ArgumentName = "userId";
+
+        // Set the 'subscriptionId' argument name to the 'id' query string parameter that represents the subscription ID
+        RestApiParameter idQueryStringParameter = getProductFromCartOperation.Parameters.Single(p => p.Location == RestApiParameterLocation.Query && p.Name == "id");
+        idQueryStringParameter.ArgumentName = "subscriptionId";
+
+        // Set the 'sessionId' argument name to the 'id' header parameter that represents the session ID
+        RestApiParameter sessionIdHeaderParameter = getProductFromCartOperation.Parameters.Single(p => p.Location == RestApiParameterLocation.Header && p.Name == "id");
+        sessionIdHeaderParameter.ArgumentName = "sessionId";
+
+        // Import the transformed OpenAPI plugin specification
+        KernelPlugin plugin = this._kernel.ImportPluginFromOpenApi("Products_Plugin", specification, new OpenApiFunctionExecutionParameters(this._httpClient));
+
+        // Create arguments for the 'addProductToCart' operation using the new argument names defined earlier.
+        // Internally these will be mapped to the correct entity when invoking the Open API endpoint.
+        KernelArguments arguments = new()
+        {
+            ["region"] = "en",
+            ["subscriptionId"] = "subscription-12345",
+            ["userId"] = "user-12345",
+            ["sessionId"] = "session-12345",
+        };
+
+        // Invoke the 'addProductToCart' function
+        await this._kernel.InvokeAsync(plugin["getProductFromCart"], arguments);
+
+        // The REST API request details
+        // {  
+        //     "RequestUri": "https://api.example.com:443/eu/users/user-12345/cart?id=subscription-12345",  
+        //     "Method": "Get", 
+        //     "Headers": {  
+        //         "id": ["session-12345"]  
+        //     }  
+        // }  
+    }
+
+    private sealed class StubHttpHandler : DelegatingHandler
+    {
+        private readonly Action<string> _requestHandler;
+        private readonly JsonSerializerOptions _options;
+
+        public StubHttpHandler(Action<string> requestHandler) : base()
+        {
+            this._requestHandler = requestHandler;
+            this._options = new JsonSerializerOptions { WriteIndented = true };
+        }
+
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            var requestData = new Dictionary<string, object>
+            {
+                { "RequestUri", request.RequestUri! },
+                { "Method", request.Method },
+                { "Headers", request.Headers.ToDictionary(h => h.Key, h => h.Value) },
+            };
+
+            this._requestHandler(JsonSerializer.Serialize(requestData, this._options));
+
+            return new HttpResponseMessage(System.Net.HttpStatusCode.OK)
+            {
+                Content = new StringContent("Success", System.Text.Encoding.UTF8, "application/json")
+            };
+        }
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        base.Dispose(disposing);
+        this._httpClient.Dispose();
+    }
+}

dotnet/samples/Concepts/Resources/Plugins/ProductsPlugin/openapi.json

@@ -0,0 +1,62 @@
+{
+  "openapi": "3.0.1",
+  "info": {
+    "title": "User Product API",
+    "version": "1.0.0",
+    "description": "API for managing products associated with users"
+  },
+  "servers": [
+    {
+      "url": "https://api.example.com/{id}",
+      "variables": {
+        "id": {
+          "default": "eu",
+          "description": "Server variable representing the region of the API (e.g., 'us' for United States, 'eu' for Europe)"
+        }
+      }
+    }
+  ],
+  "paths": {
+    "/users/{id}/cart": {
+      "get": {
+        "operationId": "getProductFromCart",
+        "summary": "Retrieve a user's cart",
+        "description": "Retrieve the contents of the cart for the user ID provided in the query parameter.",
+        "parameters": [
+          {
+            "name": "id",
+            "in": "query",
+            "required": true,
+            "description": "The ID of the subscription to retrieve products from",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "id",
+            "in": "path",
+            "required": true,
+            "description": "The ID of the user whose cart is being retrieved (query parameter)",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "id",
+            "in": "header",
+            "required": true,
+            "description": "The ID representing the session (header parameter)",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successfully retrieved the user's cart"
+          }
+        }
+      }
+    }
+  }
+}  

dotnet/src/Functions/Functions.OpenApi/Extensions/OpenApiFunctionExecutionParameters.cs

@@ -41,16 +41,19 @@ public class OpenApiFunctionExecutionParameters
 
     /// <summary>
     /// Determines whether the REST API operation payload is constructed dynamically based on payload metadata.
-    /// If false, the payload must be provided via the 'payload' argument.
+    /// It's enabled by default and allows to support operations with simple payload structure - no properties with the same name at different levels.
+    /// To support more complex payloads, it should be disabled and the payload should be provided via the 'payload' argument.
+    /// See the 'Providing Payload for OpenAPI Functions' ADR for more details: https://github.com/microsoft/semantic-kernel/blob/main/docs/decisions/0062-open-api-payload.md
     /// </summary>
     public bool EnableDynamicPayload { get; set; }
 
     /// <summary>
-    /// Determines whether payload parameter names are augmented with namespaces.
+    /// Determines whether payload parameter names are augmented with namespaces. It's only applicable when EnableDynamicPayload property is set to true.
     /// Namespaces prevent naming conflicts by adding the parent parameter name as a prefix, separated by dots.
     /// For instance, without namespaces, the 'email' parameter for both the 'sender' and 'receiver' parent parameters
     /// would be resolved from the same 'email' argument, which is incorrect. However, by employing namespaces,
     /// the parameters 'sender.email' and 'sender.receiver' will be correctly resolved from arguments with the same names.
+    /// See the 'Providing Payload for OpenAPI Functions' ADR for more details: https://github.com/microsoft/semantic-kernel/blob/main/docs/decisions/0062-open-api-payload.md
     /// </summary>
     public bool EnablePayloadNamespacing { get; set; }