diff --git a/Microsoft.Mcp.slnx b/Microsoft.Mcp.slnx index 3dad924bb2..621e7b7e8c 100644 --- a/Microsoft.Mcp.slnx +++ b/Microsoft.Mcp.slnx @@ -40,6 +40,13 @@ + + + + + + + diff --git a/eng/scripts/Measure-McpOutputSizes.ps1 b/eng/scripts/Measure-McpOutputSizes.ps1 new file mode 100644 index 0000000000..29955f52bc --- /dev/null +++ b/eng/scripts/Measure-McpOutputSizes.ps1 @@ -0,0 +1,625 @@ +#!/bin/env pwsh +#Requires -Version 7 + +<# +.SYNOPSIS + Builds and runs the MCP output size measurement test, then summarizes the results. + +.DESCRIPTION + Runs the full measurement workflow end to end: + + 1. Builds the Azure.Mcp.Server project and the standalone McpOutputSizeMeasurer tool. + 2. Runs McpOutputSizeMeasurer, which starts the MCP server over stdio in both + consolidated and namespace modes and measures the initialize greeting, + tools/list discovery, and learn-mode responses (including inner commands) for + every tool. + 3. Summarizes the report to produce console, JSON, and Markdown summaries, + extract readable description text, and split each top tool's inner commands + into individual files. + +.PARAMETER OutputDirectory + Directory for the measurement report and its artifacts. + Defaults to `/.work/mcp-output-size`. + +.PARAMETER Configuration + The build configuration to use. Defaults to `Debug`. + +.PARAMETER SkipBuild + Skip the build step and use the existing binaries. Implied (and not required) when + -ServerExecutable or -ReleaseTag is supplied, since that server binary isn't built by + this script. + +.PARAMETER Clean + Remove the output directory before running so stale artifacts are not mixed with + the new results. + +.PARAMETER LearnResponseThresholdUtf8Bytes + Include every learn response over this UTF-8 byte threshold in the summary. + Defaults to 45000. + +.PARAMETER ServerExecutable + Path to an already-built or published azmcp server executable to measure, instead of + building and using the local servers/Azure.Mcp.Server/src project. Use this to measure + a previously released version of the server (e.g. a binary extracted from a release + asset or installed via a package manager) so its output sizes can be diffed against + the current source tree. When supplied, the local server project is not built or + resolved; only the McpOutputSizeMeasurer tool is still built (unless -SkipBuild is + also passed). Mutually exclusive with -ReleaseTag. + +.PARAMETER ReleaseTag + A GitHub release tag from the microsoft/mcp repository (e.g. + `Azure.Mcp.Server-3.0.0-beta.36`) whose azmcp server asset should be downloaded and + measured, instead of building the local source tree. The matching platform zip + (`Azure.Mcp.Server--.zip`) is downloaded to + `/release-download/` and extracted before measurement. + Mutually exclusive with -ServerExecutable. + +.PARAMETER GitHubRepository + The `owner/repo` used to resolve -ReleaseTag. Defaults to `microsoft/mcp`. Only used + when -ReleaseTag is supplied. + +.EXAMPLE + ./eng/scripts/Measure-McpOutputSizes.ps1 + + Builds, measures, and summarizes using default paths. + +.EXAMPLE + ./eng/scripts/Measure-McpOutputSizes.ps1 -SkipBuild -Clean + + Reuses the existing build, clears previous results, then measures and summarizes. + +.EXAMPLE + ./eng/scripts/Measure-McpOutputSizes.ps1 -ServerExecutable C:\releases\azmcp-1.2.3\azmcp.exe -OutputDirectory .work/mcp-output-size/released-1.2.3 + + Measures a previously released azmcp build (e.g. downloaded/extracted from a GitHub + release) instead of the local source tree, writing results to a separate directory so + they can be compared against a current-source run. + +.EXAMPLE + ./eng/scripts/Measure-McpOutputSizes.ps1 -ReleaseTag Azure.Mcp.Server-3.0.0-beta.36 -OutputDirectory .work/mcp-output-size/beta.36 + + Downloads the azmcp server asset for the given release tag from GitHub, measures it, + and writes results to a separate directory so they can be compared against a + current-source run. +#> + +[CmdletBinding()] +param( + [string]$OutputDirectory, + [string]$Configuration = 'Debug', + [switch]$SkipBuild, + [switch]$Clean, + [int]$LearnResponseThresholdUtf8Bytes = 45000, + [string]$ServerExecutable, + [string]$ReleaseTag, + [string]$GitHubRepository = 'microsoft/mcp' +) + +if ($ServerExecutable -and $ReleaseTag) { + Write-Error "-ServerExecutable and -ReleaseTag are mutually exclusive." + exit 1 +} + +$ErrorActionPreference = 'Stop' +$maximumToolsPerClientRequest = 128 + +. "$PSScriptRoot/../common/scripts/common.ps1" +$repoRoot = $RepoRoot.Path + +$serverProject = Join-Path $repoRoot 'servers/Azure.Mcp.Server/src' +$measurerProject = Join-Path $repoRoot 'eng/tools/McpOutputSizeMeasurer/src' + +function Invoke-McpOutputSizeSummary { + param( + [Parameter(Mandatory)] + [string] $InputPath, + + [string] $OutputPath, + + [string] $MarkdownPath, + + [int] $LearnResponseThresholdUtf8Bytes = 45000 + ) + + if (!(Test-Path -LiteralPath $InputPath -PathType Leaf)) { + throw "Measurement report not found: $InputPath" + } + + $report = Get-Content -LiteralPath $InputPath -Raw | ConvertFrom-Json + $modeResults = @{} + foreach ($mode in $report.modes) { + $modeResults[$mode.mode] = $mode + } + + foreach ($modeName in @('consolidated', 'namespace')) { + if (!$modeResults.ContainsKey($modeName)) { + throw "The report does not contain the '$modeName' mode." + } + } + + function Get-PercentDifference([double] $value, [double] $baseline) { + if ($baseline -eq 0) { + return $null + } + + return [math]::Round((($value - $baseline) / $baseline) * 100, 2) + } + + function Get-ModeSummary($mode) { + $learnCount = @($mode.learnResponses).Count + $discoveryCount = @($mode.discoveryResponses).Count + $averageLearnBytes = if ($learnCount -eq 0) { 0 } else { + [math]::Round($mode.learnTotalUtf8Bytes / $learnCount, 2) + } + + return [ordered]@{ + mode = $mode.mode + toolCount = $mode.toolCount + greetingUtf8Bytes = $mode.initialGreetingResponse.utf8Bytes + discoveryMessageCount = $discoveryCount + discoveryUtf8Bytes = $mode.discoveryTotalUtf8Bytes + learnMessageCount = $learnCount + learnUtf8Bytes = $mode.learnTotalUtf8Bytes + averageLearnUtf8Bytes = $averageLearnBytes + totalUtf8Bytes = $mode.totalUtf8Bytes + } + } + + $consolidated = Get-ModeSummary $modeResults['consolidated'] + $namespace = Get-ModeSummary $modeResults['namespace'] + + $comparisonMetrics = @( + 'toolCount', + 'greetingUtf8Bytes', + 'discoveryMessageCount', + 'discoveryUtf8Bytes', + 'learnMessageCount', + 'learnUtf8Bytes', + 'averageLearnUtf8Bytes', + 'totalUtf8Bytes' + ) + + $comparison = [ordered]@{} + foreach ($metric in $comparisonMetrics) { + $consolidatedValue = [double]$consolidated[$metric] + $namespaceValue = [double]$namespace[$metric] + $comparison[$metric] = [ordered]@{ + consolidated = $consolidatedValue + namespace = $namespaceValue + difference = $consolidatedValue - $namespaceValue + percentDifferenceFromNamespace = Get-PercentDifference $consolidatedValue $namespaceValue + } + } + + function Save-LearnResponseText($entries) { + foreach ($entry in $entries) { + if (!$entry.learnResponseFile -or !(Test-Path -LiteralPath $entry.learnResponseFile -PathType Leaf)) { + continue + } + + $learnJson = Get-Content -LiteralPath $entry.learnResponseFile -Raw | ConvertFrom-Json + $textParts = @( + $learnJson.result.content | + Where-Object { $_.type -eq 'text' -and $null -ne $_.text } | + ForEach-Object { $_.text } + ) + + if ($textParts.Count -eq 0) { + continue + } + + $textPath = [IO.Path]::ChangeExtension($entry.learnResponseFile, '.txt') + Set-Content -LiteralPath $textPath -Value ($textParts -join "`r`n`r`n") -Encoding utf8NoBOM + $entry | Add-Member -NotePropertyName learnResponseTextFile -NotePropertyValue $textPath -Force + } + } + + function Save-InnerCommands($entries) { + foreach ($entry in $entries) { + $entry | Add-Member -NotePropertyName innerCommandCount -NotePropertyValue 0 -Force + $entry | Add-Member -NotePropertyName innerCommandDirectory -NotePropertyValue $null -Force + + if (!$entry.learnResponseTextFile -or !(Test-Path -LiteralPath $entry.learnResponseTextFile -PathType Leaf)) { + continue + } + + $text = Get-Content -LiteralPath $entry.learnResponseTextFile -Raw + $start = $text.IndexOf('[') + if ($start -lt 0) { + continue + } + + try { + $commands = @($text.Substring($start) | ConvertFrom-Json) + } catch { + Write-Warning "Could not parse inner commands for '$($entry.tool)': $_" + continue + } + + if ($commands.Count -eq 0) { + continue + } + + $toolDirectory = [IO.Path]::Combine( + [IO.Path]::GetDirectoryName($entry.learnResponseTextFile), + [IO.Path]::GetFileNameWithoutExtension($entry.learnResponseTextFile) + '-commands') + New-Item -ItemType Directory -Path $toolDirectory -Force | Out-Null + + foreach ($command in $commands) { + $commandName = if ($command.command) { $command.command } else { 'unnamed' } + foreach ($invalid in [IO.Path]::GetInvalidFileNameChars()) { + $commandName = $commandName.Replace($invalid, '-') + } + + $commandPath = [IO.Path]::Combine($toolDirectory, "$commandName.json") + Set-Content -LiteralPath $commandPath -Value ($command | ConvertTo-Json -Depth 30) -Encoding utf8NoBOM + } + + $entry.innerCommandCount = $commands.Count + $entry.innerCommandDirectory = $toolDirectory + } + } + + $allEntriesByMode = [ordered]@{ + consolidated = @($modeResults['consolidated'].learnResponses) + namespace = @($modeResults['namespace'].learnResponses) + } + foreach ($modeName in @('consolidated', 'namespace')) { + foreach ($entry in $allEntriesByMode[$modeName]) { + $entry | Add-Member -NotePropertyName learnResponseTextFile -NotePropertyValue $null -Force + } + Save-LearnResponseText $allEntriesByMode[$modeName] + Save-InnerCommands $allEntriesByMode[$modeName] + } + + function Get-TopLearnResponses($entries) { + return @($entries) | + Sort-Object -Property utf8Bytes -Descending | + Select-Object -First 10 | + ForEach-Object { + [ordered]@{ + tool = $_.tool + utf8Bytes = $_.utf8Bytes + characterCount = $_.characterCount + learnResponseFile = $_.learnResponseFile + learnResponseTextFile = $_.learnResponseTextFile + innerCommandCount = $_.innerCommandCount + innerCommandDirectory = $_.innerCommandDirectory + } + } + } + + function Get-LargeLearnResponses($entries) { + return @($entries) | + Where-Object { $_.utf8Bytes -gt $LearnResponseThresholdUtf8Bytes } | + Sort-Object -Property utf8Bytes -Descending | + ForEach-Object { + [ordered]@{ + tool = $_.tool + utf8Bytes = $_.utf8Bytes + characterCount = $_.characterCount + learnResponseFile = $_.learnResponseFile + learnResponseTextFile = $_.learnResponseTextFile + innerCommandCount = $_.innerCommandCount + innerCommandDirectory = $_.innerCommandDirectory + } + } + } + + $topLearnByMode = [ordered]@{ + consolidated = Get-TopLearnResponses $allEntriesByMode['consolidated'] + namespace = Get-TopLearnResponses $allEntriesByMode['namespace'] + } + + $largeLearnByMode = [ordered]@{ + consolidated = Get-LargeLearnResponses $allEntriesByMode['consolidated'] + namespace = Get-LargeLearnResponses $allEntriesByMode['namespace'] + } + + $summary = [ordered]@{ + sourceReport = (Resolve-Path -LiteralPath $InputPath).Path + generatedAtUtc = [DateTimeOffset]::UtcNow + transport = $report.transport + learnResponseThresholdUtf8Bytes = $LearnResponseThresholdUtf8Bytes + modes = @($consolidated, $namespace) + comparison = $comparison + topLearnResponses = $topLearnByMode + largeLearnResponses = $largeLearnByMode + } + + Write-Host "MCP output size summary ($($report.transport))" + Write-Host "" + $consoleRows = @($consolidated, $namespace) | ForEach-Object { + [pscustomobject]@{ + mode = $_['mode'] + toolCount = $_['toolCount'] + greetingUtf8Bytes = $_['greetingUtf8Bytes'] + discoveryUtf8Bytes = $_['discoveryUtf8Bytes'] + learnUtf8Bytes = $_['learnUtf8Bytes'] + totalUtf8Bytes = $_['totalUtf8Bytes'] + } + } + $consoleRows | + Format-Table mode, toolCount, greetingUtf8Bytes, discoveryUtf8Bytes, learnUtf8Bytes, totalUtf8Bytes | + Out-Host + + Write-Host "Consolidated relative to namespace:" + foreach ($metric in $comparisonMetrics) { + $result = $comparison[$metric] + $percent = if ($null -eq $result.percentDifferenceFromNamespace) { + 'n/a' + } else { + "$($result.percentDifferenceFromNamespace)%" + } + Write-Host (" {0}: difference {1}, {2}" -f $metric, $result.difference, $percent) + } + + foreach ($modeName in @('consolidated', 'namespace')) { + Write-Host "" + Write-Host "Top 10 largest learn responses ($modeName):" + $topLearnByMode[$modeName] | ForEach-Object { + [pscustomobject]@{ + tool = $_.tool + utf8Bytes = $_.utf8Bytes + innerCommands = $_.innerCommandCount + } + } | Format-Table tool, utf8Bytes, innerCommands | Out-Host + + Write-Host "Learn responses over $LearnResponseThresholdUtf8Bytes UTF-8 bytes ($modeName):" + $largeLearnByMode[$modeName] | ForEach-Object { + [pscustomobject]@{ + tool = $_.tool + utf8Bytes = $_.utf8Bytes + } + } | Format-Table tool, utf8Bytes | Out-Host + } + + $summaryJson = $summary | ConvertTo-Json -Depth 10 + if (![string]::IsNullOrWhiteSpace($OutputPath)) { + $resolvedOutputPath = [IO.Path]::GetFullPath($OutputPath) + $outputDirectory = [IO.Path]::GetDirectoryName($resolvedOutputPath) + if (![string]::IsNullOrWhiteSpace($outputDirectory)) { + New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null + } + Set-Content -LiteralPath $resolvedOutputPath -Value $summaryJson -Encoding utf8 + Write-Host "" + Write-Host "Summary JSON saved to $resolvedOutputPath" + } else { + Write-Output $summaryJson + } + + if ([string]::IsNullOrWhiteSpace($MarkdownPath)) { + $MarkdownPath = [IO.Path]::ChangeExtension( + [IO.Path]::GetFullPath($InputPath), + '.md') + } + + $markdownLines = [System.Collections.Generic.List[string]]::new() + $markdownLines.Add('# MCP Output Size Summary') + $markdownLines.Add('') + $markdownLines.Add('- **Source report:** `' + $summary.sourceReport + '`') + $markdownLines.Add("- **Transport:** $($summary.transport)") + $markdownLines.Add("- **Generated:** $($summary.generatedAtUtc)") + $markdownLines.Add("- **Large learn response threshold:** $($summary.learnResponseThresholdUtf8Bytes) UTF-8 bytes") + $markdownLines.Add('') + $markdownLines.Add('## Mode Summary') + $markdownLines.Add('') + $markdownLines.Add('| Mode | Tools | Greeting (bytes) | Discovery (bytes) | Learn (bytes) | Total (bytes) |') + $markdownLines.Add('| --- | ---: | ---: | ---: | ---: | ---: |') + foreach ($mode in $summary.modes) { + $markdownLines.Add( + "| $($mode.mode) | $($mode.toolCount) | $($mode.greetingUtf8Bytes) | " + + "$($mode.discoveryUtf8Bytes) | $($mode.learnUtf8Bytes) | $($mode.totalUtf8Bytes) |") + } + + $markdownLines.Add('') + $markdownLines.Add('## Consolidated vs. Namespace') + $markdownLines.Add('') + $markdownLines.Add('| Metric | Consolidated | Namespace | Difference | Difference vs. namespace |') + $markdownLines.Add('| --- | ---: | ---: | ---: | ---: |') + foreach ($metric in $comparisonMetrics) { + $result = $comparison[$metric] + $percent = if ($null -eq $result.percentDifferenceFromNamespace) { + 'n/a' + } else { + "$($result.percentDifferenceFromNamespace)%" + } + $markdownLines.Add( + "| $metric | $($result.consolidated) | $($result.namespace) | " + + "$($result.difference) | $percent |") + } + + $resolvedMarkdownPath = [IO.Path]::GetFullPath($MarkdownPath) + $markdownDirectory = [IO.Path]::GetDirectoryName($resolvedMarkdownPath) + if (![string]::IsNullOrWhiteSpace($markdownDirectory)) { + New-Item -ItemType Directory -Path $markdownDirectory -Force | Out-Null + } + + foreach ($modeName in @('consolidated', 'namespace')) { + $markdownLines.Add('') + $markdownLines.Add("## Top 10 Largest Learn Responses ($modeName)") + $markdownLines.Add('') + $markdownLines.Add('| Rank | Tool | Bytes | Inner Commands | Saved Response File | Description Text File |') + $markdownLines.Add('| ---: | --- | ---: | ---: | --- | --- |') + $rank = 1 + foreach ($entry in $topLearnByMode[$modeName]) { + $fileLink = if ($entry.learnResponseFile) { "``$($entry.learnResponseFile)``" } else { 'n/a' } + $textLink = if ($entry.learnResponseTextFile) { "``$($entry.learnResponseTextFile)``" } else { 'n/a' } + $markdownLines.Add("| $rank | $($entry.tool) | $($entry.utf8Bytes) | $($entry.innerCommandCount) | $fileLink | $textLink |") + $rank++ + } + + $markdownLines.Add('') + $markdownLines.Add("## Learn Responses Over $LearnResponseThresholdUtf8Bytes UTF-8 Bytes ($modeName)") + $markdownLines.Add('') + $markdownLines.Add('| Tool | Bytes | Character Count | Saved Response File | Description Text File |') + $markdownLines.Add('| --- | ---: | ---: | --- | --- |') + foreach ($entry in $largeLearnByMode[$modeName]) { + $fileLink = if ($entry.learnResponseFile) { "``$($entry.learnResponseFile)``" } else { 'n/a' } + $textLink = if ($entry.learnResponseTextFile) { "``$($entry.learnResponseTextFile)``" } else { 'n/a' } + $markdownLines.Add("| $($entry.tool) | $($entry.utf8Bytes) | $($entry.characterCount) | $fileLink | $textLink |") + } + } + + Set-Content -LiteralPath $resolvedMarkdownPath -Value $markdownLines -Encoding utf8 + Write-Host "Markdown summary saved to $resolvedMarkdownPath" +} + +if (!$OutputDirectory) { + $OutputDirectory = Join-Path $repoRoot '.work/mcp-output-size' +} +$OutputDirectory = [IO.Path]::GetFullPath($OutputDirectory) + +if ($Clean -and (Test-Path -LiteralPath $OutputDirectory)) { + Write-Host "Removing existing output directory $OutputDirectory" + Remove-Item -LiteralPath $OutputDirectory -Recurse -Force +} + +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +$reportPath = Join-Path $OutputDirectory 'mcp-output-size.json' + +if ($ServerExecutable) { + $requestedServerExecutable = [IO.Path]::GetFullPath($ServerExecutable) + if (!(Test-Path -LiteralPath $requestedServerExecutable -PathType Leaf)) { + Write-Error "Server executable not found at $requestedServerExecutable." + exit 1 + } +} + +$resolvedServerExecutable = $null + +if ($ReleaseTag) { + # Map the current platform to the asset name pattern used by Pack-Zip.ps1 / + # New-BuildInfo.ps1, e.g. Azure.Mcp.Server-win-x64.zip, Azure.Mcp.Server-linux-arm64.zip, + # Azure.Mcp.Server-osx-arm64.zip. + if ($IsWindows) { + $releaseOs = 'win' + } elseif ($IsMacOS) { + $releaseOs = 'osx' + } elseif ($IsLinux) { + $releaseOs = 'linux' + } else { + Write-Error "Unable to determine current OS for release asset selection." + exit 1 + } + + $arch = [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture + $releaseArch = switch ($arch) { + 'X64' { 'x64' } + 'Arm64' { 'arm64' } + default { + Write-Error "Unsupported process architecture '$arch' for release asset selection." + exit 1 + } + } + + $assetName = "Azure.Mcp.Server-$releaseOs-$releaseArch.zip" + $downloadUrl = "https://github.com/$GitHubRepository/releases/download/$ReleaseTag/$assetName" + + $releaseDownloadDir = Join-Path $OutputDirectory "release-download/$ReleaseTag" + $releaseExtractDir = Join-Path $releaseDownloadDir 'extracted' + $releaseZipPath = Join-Path $releaseDownloadDir $assetName + + New-Item -ItemType Directory -Path $releaseDownloadDir -Force | Out-Null + + Write-Host "Downloading $assetName from release $ReleaseTag ($GitHubRepository)..." + Write-Host " $downloadUrl" + Invoke-WebRequest -Uri $downloadUrl -OutFile $releaseZipPath + + Write-Host "Extracting $releaseZipPath..." + if (Test-Path -LiteralPath $releaseExtractDir) { + Remove-Item -LiteralPath $releaseExtractDir -Recurse -Force + } + Expand-Archive -Path $releaseZipPath -DestinationPath $releaseExtractDir -Force + + $resolvedServerExecutable = Join-Path $releaseExtractDir "azmcp$(if ($IsWindows) { '.exe' } else { '' })" + if (!(Test-Path -LiteralPath $resolvedServerExecutable -PathType Leaf)) { + Write-Error "azmcp executable not found in downloaded release asset at $resolvedServerExecutable." + exit 1 + } + if (!$IsWindows) { + chmod +x $resolvedServerExecutable + } +} + +if (-not $resolvedServerExecutable -and $requestedServerExecutable) { + $resolvedServerExecutable = $requestedServerExecutable +} + +$usingExternalServerBinary = [bool]$resolvedServerExecutable + +if ($SkipBuild) { + Write-Host "Skipping build." +} else { + if ($usingExternalServerBinary) { + Write-Host "Using pre-built server executable $resolvedServerExecutable; skipping local server build." + } else { + Write-Host "Building $serverProject ($Configuration)..." + dotnet build $serverProject --configuration $Configuration + if ($LASTEXITCODE -ne 0) { + Write-Error "Build failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } + + Write-Host "Building $measurerProject ($Configuration)..." + dotnet build $measurerProject --configuration $Configuration + if ($LASTEXITCODE -ne 0) { + Write-Error "Build failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } +} + +if ($usingExternalServerBinary) { + $serverExecutablePath = $resolvedServerExecutable +} else { + $serverExecutablePath = Join-Path $serverProject "bin/$Configuration/net10.0/azmcp$(if ($IsWindows) { '.exe' } else { '' })" + if (!(Test-Path -LiteralPath $serverExecutablePath -PathType Leaf)) { + Write-Error "Server executable not found at $serverExecutablePath. Run without -SkipBuild to build it first." + exit 1 + } +} + +$measurerExecutable = Join-Path $measurerProject "bin/$Configuration/net10.0/McpOutputSizeMeasurer$(if ($IsWindows) { '.exe' } else { '' })" +if (!(Test-Path -LiteralPath $measurerExecutable -PathType Leaf)) { + Write-Error "Measurer executable not found at $measurerExecutable. Run without -SkipBuild to build it first." + exit 1 +} + +Write-Host "Running MCP output size measurement..." +& $measurerExecutable --executable $serverExecutablePath --report $reportPath +if ($LASTEXITCODE -ne 0) { + Write-Error "Measurement failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE +} + +if (!(Test-Path -LiteralPath $reportPath -PathType Leaf)) { + Write-Error "Measurement report was not produced at $reportPath." + exit 1 +} + +Write-Host "Validating MCP tool counts against the maximum supported client request size ($maximumToolsPerClientRequest)..." +$measurementReport = Get-Content -LiteralPath $reportPath -Raw | ConvertFrom-Json +$exceedingModes = @( + $measurementReport.modes | + Where-Object { [int]$_.toolCount -gt $maximumToolsPerClientRequest } +) +if ($exceedingModes.Count -gt 0) { + $details = $exceedingModes | + ForEach-Object { "'$($_.mode)' has $($_.toolCount) tools" } | + Join-String -Separator ', ' + $limitMessage = "MCP tool count exceeds the maximum supported client request size of ${maximumToolsPerClientRequest}: $details." + Write-Host "" + Write-Host "ERROR: $limitMessage" -ForegroundColor Red + Write-Warning $limitMessage +} + +Write-Host "Summarizing results..." +Invoke-McpOutputSizeSummary ` + -InputPath $reportPath ` + -OutputPath (Join-Path $OutputDirectory 'mcp-output-size-summary.json') ` + -MarkdownPath (Join-Path $OutputDirectory 'mcp-output-size-summary.md') ` + -LearnResponseThresholdUtf8Bytes $LearnResponseThresholdUtf8Bytes + +Write-Host "" +Write-Host "Done. Results are in $OutputDirectory" diff --git a/eng/tools/McpOutputSizeMeasurer/src/McpOutputSizeMeasurer.cs b/eng/tools/McpOutputSizeMeasurer/src/McpOutputSizeMeasurer.cs new file mode 100644 index 0000000000..90ae43f24e --- /dev/null +++ b/eng/tools/McpOutputSizeMeasurer/src/McpOutputSizeMeasurer.cs @@ -0,0 +1,389 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Text; +using System.Text.Json; + +namespace McpOutputSizeMeasurer; + +/// +/// Measures MCP responses from a client perspective for a server's exposed tool surfaces. +/// Starts the given azmcp-compatible executable over stdio, walks tool discovery +/// (tools/list, paginated), calls every tool's learn-mode response, and re-queries every +/// inner command a tool's learn response advertises. All requests/responses are exchanged +/// as raw JSON-RPC text so that UTF-8 byte counts reflect exactly what crosses the wire. +/// +public sealed class McpOutputSizeMeasurer +{ + private readonly Action? _logger; + + public McpOutputSizeMeasurer(Action? logger = null) + { + _logger = logger; + } + + /// + /// Runs the full measurement workflow for every mode in and + /// returns a report object with the same shape previously produced by + /// Report shape: { transport, generatedAtUtc, reportPath, modes }. + /// + public async Task MeasureAsync( + string executablePath, + IReadOnlyList modes, + string reportDirectory, + string reportPath, + CancellationToken cancellationToken = default) + { + var measurements = new List(); + foreach (var mode in modes) + { + measurements.Add(await MeasureModeAsync(executablePath, mode, reportDirectory, cancellationToken)); + } + + return new + { + transport = "stdio", + generatedAtUtc = DateTimeOffset.UtcNow, + reportPath, + modes = measurements + }; + } + + public async Task MeasureModeAsync( + string executablePath, + string mode, + string reportDirectory, + CancellationToken cancellationToken = default) + { + if (!File.Exists(executablePath)) + { + throw new FileNotFoundException( + $"Executable not found at {executablePath}. Build the server project first.", + executablePath); + } + + using var process = Process.Start(new ProcessStartInfo + { + FileName = executablePath, + Arguments = $"server start --mode {mode}", + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = false, + CreateNoWindow = true + }) ?? throw new InvalidOperationException($"Failed to start process for {executablePath}."); + + try + { + await Task.Delay(500, cancellationToken); + + var initializeResponse = await SendRequestAsync( + process, + """ + {"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"mcp-output-size-measurer","version":"1.0"}}} + """, + cancellationToken); + + await SendNotificationAsync( + process, + """{"jsonrpc":"2.0","method":"notifications/initialized"}""", + cancellationToken); + + var discoveryResponses = new List(); + var discoveryTexts = new List(); + var tools = new List(); + var discoveryTotalUtf8Bytes = 0; + string? cursor = null; + var requestId = 2; + + do + { + var request = cursor is null + ? JsonSerializer.Serialize(new + { + jsonrpc = "2.0", + id = requestId, + method = "tools/list", + @params = new { } + }) + : JsonSerializer.Serialize(new + { + jsonrpc = "2.0", + id = requestId, + method = "tools/list", + @params = new { cursor } + }); + var response = await SendRequestAsync(process, request, cancellationToken); + discoveryTexts.Add(response); + using var document = JsonDocument.Parse(response); + var result = document.RootElement.GetProperty("result"); + foreach (var tool in result.GetProperty("tools").EnumerateArray()) + { + tools.Add(tool.GetProperty("name").GetString()!); + } + + cursor = result.TryGetProperty("nextCursor", out var nextCursor) + ? nextCursor.GetString() + : null; + discoveryTotalUtf8Bytes += GetUtf8ByteCount(response); + discoveryResponses.Add(new + { + messageNumber = discoveryResponses.Count + 1, + utf8Bytes = GetUtf8ByteCount(response), + characterCount = response.Length, + toolCount = result.GetProperty("tools").GetArrayLength() + }); + requestId++; + } + while (cursor is not null); + + if (tools.Count == 0) + { + throw new InvalidOperationException($"No tools were discovered for mode '{mode}'."); + } + + // Save every tool's learn response in a per-mode subdirectory so any payload can be + // inspected without re-running the server. The directory is cleared first so stale + // files from a previous run (e.g. a tool that no longer produces the largest + // response) don't linger alongside the current run's files. Filtering down to the + // largest/most interesting responses is left to the summarization script, which + // operates on this full, unfiltered set. + var learnDirectory = Path.Combine(reportDirectory, mode); + if (Directory.Exists(learnDirectory)) + { + Directory.Delete(learnDirectory, recursive: true); + } + Directory.CreateDirectory(learnDirectory); + + var learnResponses = new List(tools.Count); + var learnResponseTextByTool = new Dictionary(tools.Count, StringComparer.Ordinal); + var learnTotalUtf8Bytes = 0; + foreach (var tool in tools) + { + var request = JsonSerializer.Serialize(new + { + jsonrpc = "2.0", + id = requestId, + method = "tools/call", + @params = new + { + name = tool, + arguments = new + { + intent = "Measure available commands", + learn = true + } + } + }); + var response = await SendRequestAsync(process, request, cancellationToken); + using var document = JsonDocument.Parse(response); + if (!document.RootElement.TryGetProperty("result", out _)) + { + throw new InvalidOperationException($"The learn response for '{tool}' did not contain a result."); + } + + learnTotalUtf8Bytes += GetUtf8ByteCount(response); + learnResponseTextByTool[tool] = response; + + var learnResponseFile = Path.Combine( + learnDirectory, + $"{SanitizeFileNameSegment(tool)}.json"); + await File.WriteAllTextAsync(learnResponseFile, response, Encoding.UTF8, cancellationToken); + + learnResponses.Add(new + { + tool, + utf8Bytes = GetUtf8ByteCount(response), + characterCount = response.Length, + learnResponseFile + }); + requestId++; + } + + // Verify that per-command learn requests also return details, mirroring the + // top-level tool discovery. Each inner command advertised by a tool's learn + // response is re-requested with "command" set and "learn" still true. + var commandLearnResponses = new List(); + var commandLearnTotalUtf8Bytes = 0; + foreach (var tool in tools) + { + foreach (var command in GetInnerCommandNames(learnResponseTextByTool[tool])) + { + var request = JsonSerializer.Serialize(new + { + jsonrpc = "2.0", + id = requestId, + method = "tools/call", + @params = new + { + name = tool, + arguments = new + { + intent = "Measure command details", + command, + learn = true, + parameters = new { } + } + } + }); + var response = await SendRequestAsync(process, request, cancellationToken); + using var document = JsonDocument.Parse(response); + if (!document.RootElement.TryGetProperty("result", out var commandResult)) + { + throw new InvalidOperationException($"The learn response for '{tool}.{command}' did not contain a result."); + } + if (!commandResult.TryGetProperty("content", out var commandContent) || + commandContent.GetArrayLength() == 0) + { + throw new InvalidOperationException($"The learn response for '{tool}.{command}' did not contain any content."); + } + + commandLearnTotalUtf8Bytes += GetUtf8ByteCount(response); + commandLearnResponses.Add(new + { + tool, + command, + utf8Bytes = GetUtf8ByteCount(response), + characterCount = response.Length + }); + requestId++; + } + } + + var discoveryTextPath = Path.Combine( + reportDirectory, + $"mcp-output-size-{mode}-discovery.jsonl"); + await File.WriteAllTextAsync( + discoveryTextPath, + string.Join(Environment.NewLine, discoveryTexts) + Environment.NewLine, + Encoding.UTF8, + cancellationToken); + + return new + { + mode, + toolCount = tools.Count, + initialGreetingResponse = MeasureMessage(initializeResponse), + discoveryTextFile = discoveryTextPath, + discoveryResponses, + discoveryTotalUtf8Bytes, + learnResponses, + learnTotalUtf8Bytes, + commandLearnCount = commandLearnResponses.Count, + commandLearnTotalUtf8Bytes, + commandLearnResponses, + totalUtf8Bytes = GetUtf8ByteCount(initializeResponse) + + discoveryTotalUtf8Bytes + + learnTotalUtf8Bytes + }; + } + finally + { + if (!process.HasExited) + { + process.Kill(); + } + } + } + + private async Task SendRequestAsync(Process process, string request, CancellationToken cancellationToken) + { + _logger?.Invoke($"--> {request}"); + await process.StandardInput.WriteLineAsync(request); + await process.StandardInput.FlushAsync(cancellationToken); + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + var response = await process.StandardOutput.ReadLineAsync(timeout.Token) + ?? throw new InvalidOperationException("The server closed its output stream before responding."); + _logger?.Invoke($"<-- {response}"); + return response; + } + + private static async Task SendNotificationAsync(Process process, string notification, CancellationToken cancellationToken) + { + await process.StandardInput.WriteLineAsync(notification); + await process.StandardInput.FlushAsync(cancellationToken); + } + + private static object MeasureMessage(string message) => new + { + utf8Bytes = GetUtf8ByteCount(message), + characterCount = message.Length + }; + + private static int GetUtf8ByteCount(string message) + => Encoding.UTF8.GetByteCount(message); + + private static string SanitizeFileNameSegment(string value) + { + var sanitized = value; + foreach (var invalidChar in Path.GetInvalidFileNameChars()) + { + sanitized = sanitized.Replace(invalidChar, '-'); + } + + return sanitized; + } + + /// + /// Parses the inner command names from a tool's learn response. The learn text is a short + /// preamble followed by a JSON array of command descriptors. + /// + internal static List GetInnerCommandNames(string learnResponse) + { + var commands = new List(); + + using var document = JsonDocument.Parse(learnResponse); + if (!document.RootElement.TryGetProperty("result", out var result) || + !result.TryGetProperty("content", out var content)) + { + return commands; + } + + foreach (var block in content.EnumerateArray()) + { + if (!block.TryGetProperty("text", out var textElement)) + { + continue; + } + + var text = textElement.GetString(); + var start = text?.IndexOf('[') ?? -1; + if (text is null || start < 0) + { + continue; + } + + JsonDocument commandDocument; + try + { + commandDocument = JsonDocument.Parse(text[start..]); + } + catch (JsonException) + { + continue; + } + + using (commandDocument) + { + if (commandDocument.RootElement.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var command in commandDocument.RootElement.EnumerateArray()) + { + if (command.TryGetProperty("command", out var name) && + name.GetString() is { Length: > 0 } commandName) + { + commands.Add(commandName); + } + } + } + } + + return commands; + } +} diff --git a/eng/tools/McpOutputSizeMeasurer/src/McpOutputSizeMeasurer.csproj b/eng/tools/McpOutputSizeMeasurer/src/McpOutputSizeMeasurer.csproj new file mode 100644 index 0000000000..fc3b4139f6 --- /dev/null +++ b/eng/tools/McpOutputSizeMeasurer/src/McpOutputSizeMeasurer.csproj @@ -0,0 +1,25 @@ + + + + Exe + + false + McpOutputSizeMeasurer + enable + enable + + + + + + + + + + + diff --git a/eng/tools/McpOutputSizeMeasurer/src/Program.cs b/eng/tools/McpOutputSizeMeasurer/src/Program.cs new file mode 100644 index 0000000000..4add1e9087 --- /dev/null +++ b/eng/tools/McpOutputSizeMeasurer/src/Program.cs @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text; +using System.Text.Json; +using McpToolEvaluator.Core; + +namespace McpOutputSizeMeasurer; + +/// +/// Standalone console application that measures MCP responses from a client perspective for +/// the server's exposed tool surfaces. Starts the azmcp executable over stdio in both +/// consolidated and namespace modes and measures the initialize greeting, tools/list +/// discovery, and learn-mode responses (including inner commands) for every tool, writing a +/// JSON report that Measure-McpOutputSizes.ps1 summarizes. +/// +internal static class Program +{ + private static readonly string[] DefaultModes = ["consolidated", "namespace"]; + + private static async Task Main(string[] args) + { + if (args.Contains("--help") || args.Contains("-h")) + { + ShowHelp(); + return 0; + } + + string? executablePath = null; + string? reportPath = null; + var modes = new List(); + var verbose = false; + + for (var i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--executable": + case "--exe": + executablePath = RequireValue(args, ref i, "--executable"); + break; + case "--report": + reportPath = RequireValue(args, ref i, "--report"); + break; + case "--mode": + modes.Add(RequireValue(args, ref i, "--mode")); + break; + case "--verbose": + verbose = true; + break; + default: + Console.Error.WriteLine($"Unknown argument: {args[i]}"); + return -1; + } + } + + if (modes.Count == 0) + { + modes.AddRange(DefaultModes); + } + + var repoRoot = Utilities.FindRepoRoot(AppContext.BaseDirectory); + + if (string.IsNullOrEmpty(executablePath)) + { + var executableName = OperatingSystem.IsWindows() ? "azmcp.exe" : "azmcp"; + executablePath = Path.Combine( + repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug", "net10.0", executableName); + } + + if (!File.Exists(executablePath)) + { + Console.Error.WriteLine( + $"Executable not found at {executablePath}. Build the Azure.Mcp.Server project first, or pass --executable ."); + return -1; + } + + reportPath ??= Path.Combine(repoRoot, "TestResults", "mcp-output-size.json"); + reportPath = Path.GetFullPath(reportPath); + var reportDirectory = Path.GetDirectoryName(reportPath)!; + Directory.CreateDirectory(reportDirectory); + + var logger = verbose ? (Action)Console.Error.WriteLine : null; + var measurer = new McpOutputSizeMeasurer(logger); + + object report; + try + { + report = await measurer.MeasureAsync(executablePath, modes, reportDirectory, reportPath); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Measurement failed: {ex.Message}"); + return -1; + } + + var reportJson = JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(reportPath, reportJson, Encoding.UTF8); + + Console.WriteLine(reportJson); + Console.WriteLine($"Measurement report saved to {reportPath}"); + + return 0; + } + + private static string RequireValue(string[] args, ref int index, string optionName) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException($"Missing value for {optionName}."); + } + + index++; + return args[index]; + } + + private static void ShowHelp() + { + Console.WriteLine(""" + MCP Output Size Measurer + + Measures MCP responses from a client perspective for the server's exposed tool + surfaces: the initialize greeting, tools/list discovery, and learn-mode responses + (including inner commands) for every tool, in one or more server modes. + + Usage: + McpOutputSizeMeasurer [options] + + Options: + --executable Path to the azmcp(.exe) executable to measure. + Defaults to servers/Azure.Mcp.Server/src/bin/Debug/net10.0/azmcp(.exe). + --report Path to write the JSON measurement report. + Defaults to TestResults/mcp-output-size.json. + --mode Server mode to measure (repeatable). Defaults to + "consolidated" and "namespace". + --verbose Log every JSON-RPC request/response to stderr. + --help, -h Show this help. + """); + } +} diff --git a/eng/tools/McpOutputSizeMeasurer/tests/McpOutputSizeMeasurer.Tests/McpOutputSizeMeasurer.Tests.csproj b/eng/tools/McpOutputSizeMeasurer/tests/McpOutputSizeMeasurer.Tests/McpOutputSizeMeasurer.Tests.csproj new file mode 100644 index 0000000000..a133d6b554 --- /dev/null +++ b/eng/tools/McpOutputSizeMeasurer/tests/McpOutputSizeMeasurer.Tests/McpOutputSizeMeasurer.Tests.csproj @@ -0,0 +1,13 @@ + + + true + Exe + + + + + + + + + diff --git a/eng/tools/McpOutputSizeMeasurer/tests/McpOutputSizeMeasurer.Tests/McpOutputSizeMeasurerTests.cs b/eng/tools/McpOutputSizeMeasurer/tests/McpOutputSizeMeasurer.Tests/McpOutputSizeMeasurerTests.cs new file mode 100644 index 0000000000..769670170c --- /dev/null +++ b/eng/tools/McpOutputSizeMeasurer/tests/McpOutputSizeMeasurer.Tests/McpOutputSizeMeasurerTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Xunit; + +namespace McpOutputSizeMeasurer.Tests; + +public class McpOutputSizeMeasurerTests +{ + [Fact] + public void GetInnerCommandNames_ParsesCommandArrayFromLearnResponse() + { + const string learnResponse = """ + {"jsonrpc":"2.0","id":5,"result":{"content":[{"type":"text","text":"Preamble text describing the tool.\n[{\"command\":\"group_subcommand-one\",\"description\":\"first\"},{\"command\":\"group_subcommand-two\",\"description\":\"second\"}]"}]}} + """; + + var commands = McpOutputSizeMeasurer.GetInnerCommandNames(learnResponse); + + Assert.Equal(["group_subcommand-one", "group_subcommand-two"], commands); + } + + [Fact] + public void GetInnerCommandNames_ReturnsEmpty_WhenResponseHasNoResult() + { + const string learnResponse = """{"jsonrpc":"2.0","id":5,"error":{"code":-32601,"message":"not found"}}"""; + + var commands = McpOutputSizeMeasurer.GetInnerCommandNames(learnResponse); + + Assert.Empty(commands); + } + + [Fact] + public void GetInnerCommandNames_ReturnsEmpty_WhenTextHasNoJsonArray() + { + const string learnResponse = """ + {"jsonrpc":"2.0","id":5,"result":{"content":[{"type":"text","text":"No inner commands here."}]}} + """; + + var commands = McpOutputSizeMeasurer.GetInnerCommandNames(learnResponse); + + Assert.Empty(commands); + } +} diff --git a/eng/tools/Tools.sln b/eng/tools/Tools.sln index d1a4be767e..e1e5cbd827 100644 --- a/eng/tools/Tools.sln +++ b/eng/tools/Tools.sln @@ -30,6 +30,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VallyEvaluator", "VallyEval EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VallyEvaluator.Tests", "VallyEvaluator\tests\VallyEvaluator.Tests.csproj", "{62A1BDE9-3C4D-516F-8B0F-9256B60B4D1C}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "McpOutputSizeMeasurer", "McpOutputSizeMeasurer", "{A1B2C3D4-1111-4A2B-9C3D-4E5F60718293}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "McpOutputSizeMeasurer", "McpOutputSizeMeasurer\src\McpOutputSizeMeasurer.csproj", "{B2C3D4E5-2222-4B3C-8D4E-5F6071829314}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "McpOutputSizeMeasurer.Tests", "McpOutputSizeMeasurer\tests\McpOutputSizeMeasurer.Tests.csproj", "{C3D4E5F6-3333-4C4D-9E5F-607182930415}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -72,6 +78,14 @@ Global {62A1BDE9-3C4D-516F-8B0F-9256B60B4D1C}.Debug|Any CPU.Build.0 = Debug|Any CPU {62A1BDE9-3C4D-516F-8B0F-9256B60B4D1C}.Release|Any CPU.ActiveCfg = Release|Any CPU {62A1BDE9-3C4D-516F-8B0F-9256B60B4D1C}.Release|Any CPU.Build.0 = Release|Any CPU + {B2C3D4E5-2222-4B3C-8D4E-5F6071829314}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B2C3D4E5-2222-4B3C-8D4E-5F6071829314}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B2C3D4E5-2222-4B3C-8D4E-5F6071829314}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B2C3D4E5-2222-4B3C-8D4E-5F6071829314}.Release|Any CPU.Build.0 = Release|Any CPU + {C3D4E5F6-3333-4C4D-9E5F-607182930415}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3D4E5F6-3333-4C4D-9E5F-607182930415}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3D4E5F6-3333-4C4D-9E5F-607182930415}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3D4E5F6-3333-4C4D-9E5F-607182930415}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -86,6 +100,8 @@ Global {27791D6A-A5E3-8D86-29BE-FFFCB19F1431} = {2F921C00-8FF7-4FB4-99C6-3DB9C123B258} {CEC00C5C-51C4-0827-9409-40D1454CB1DF} = {F504E075-6E73-4538-859E-9D5B983CC8F7} {62A1BDE9-3C4D-516F-8B0F-9256B60B4D1C} = {F504E075-6E73-4538-859E-9D5B983CC8F7} + {B2C3D4E5-2222-4B3C-8D4E-5F6071829314} = {A1B2C3D4-1111-4A2B-9C3D-4E5F60718293} + {C3D4E5F6-3333-4C4D-9E5F-607182930415} = {A1B2C3D4-1111-4A2B-9C3D-4E5F60718293} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FE4B2877-CB8E-40CE-92EA-A30A5B9C07EE}