From ae4cb975bfa14f0cfe96893580ddf6c1bee0b224 Mon Sep 17 00:00:00 2001 From: Nikola Metulev <711864+nmetulev@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:18:16 -0700 Subject: [PATCH 1/2] Benchmark: score the winapp-performance and winui-devtools skills Add a current winapp capability map for all 14 shipped skills, keep legacy maps for older results, recognize 'winapp perf' commands, and reword the winapp-performance description so it no longer leaks a held-out fixture's wording. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 72865eb6-685a-4ede-8516-17cfe8abcc86 --- benchmarks/agents/README.md | 5 +- benchmarks/agents/capabilities.json | 108 +++++++++++++++++- benchmarks/agents/lib/Benchmark.psm1 | 4 +- benchmarks/agents/tests/Benchmark.Tests.ps1 | 5 + .../agents/tests/Capabilities.Tests.ps1 | 9 ++ .../winapp/skills/winapp-performance/SKILL.md | 2 +- 6 files changed, 126 insertions(+), 7 deletions(-) diff --git a/benchmarks/agents/README.md b/benchmarks/agents/README.md index 049e1ba2b..08dd26074 100644 --- a/benchmarks/agents/README.md +++ b/benchmarks/agents/README.md @@ -289,7 +289,7 @@ rows are more reliable. that serve it, and maps each plugin's skills to capabilities: ```json -{ "id": "winapp-current", "plugin": "winapp", "skillSetHash": "ea21288d7949", +{ "id": "winapp-current", "plugin": "winapp", "skillSetHash": "e20195ae2f87", "skills": { "winapp-signing": ["msix.sign"], "winapp-sandbox": ["sandbox.run", "ui.automate"] } } ``` @@ -331,6 +331,9 @@ forbid `winui.*`. Mapping choices that affect scores: elements). It does not provide `ui.automate`, so loading it for a click, type, read-text, or screenshot task earns no routing credit, and scenarios where it is the wrong tool forbid `winui.inspect`. Loading it in a non-WinUI scenario fails the `winui.*` forbid. +- `winapp-performance` provides `winui.build`: diagnosing a slow WinUI app is part of running and + debugging it, and `winapp perf` commands count toward it. Its scope is WinUI 3, so loading it in + a non-WinUI scenario fails the `winui.*` forbid. - Only `winui-packaging` provides `store.publish`. In the `winapp` configuration a Store scenario is `n/a`; where it is installed, a run that loads only packaging guidance scores `partial`. diff --git a/benchmarks/agents/capabilities.json b/benchmarks/agents/capabilities.json index 6be121663..1ac0ea360 100644 --- a/benchmarks/agents/capabilities.json +++ b/benchmarks/agents/capabilities.json @@ -40,9 +40,10 @@ } }, "winui.build": { - "description": "Build, run, and debug a WinUI 3 app: build errors, startup crashes, and Native AOT publish.", + "description": "Build, run, and debug a WinUI 3 app: build errors, startup crashes, slow layout or scrolling, and Native AOT publish.", "commands": [ - "^run$" + "^run$", + "^perf" ] }, "msix.package": { @@ -252,6 +253,58 @@ "id": "winapp-current", "plugin": "winapp", "source": "plugins/winapp", + "skillSetHash": "e20195ae2f87", + "skills": { + "winapp-find-api": [ + "api.lookup" + ], + "winapp-find-ui": [ + "ui.samples" + ], + "winapp-frameworks": [ + "framework.guidance" + ], + "winapp-identity": [ + "identity" + ], + "winapp-manifest": [ + "msix.manifest" + ], + "winapp-maui": [ + "maui.package" + ], + "winapp-package": [ + "msix.package" + ], + "winapp-performance": [ + "winui.build" + ], + "winapp-sandbox": [ + "sandbox.run", + "ui.automate" + ], + "winapp-setup": [ + "project.setup", + "project.scaffold" + ], + "winapp-signing": [ + "msix.sign" + ], + "winapp-troubleshoot": [ + "troubleshoot" + ], + "winapp-ui-automation": [ + "ui.automate" + ], + "winui-devtools": [ + "winui.inspect" + ] + } + }, + { + "id": "winapp-before-performance", + "plugin": "winapp", + "source": "plugins/winapp with winui-devtools, before winapp-performance was mapped", "skillSetHash": "ea21288d7949", "skills": { "winapp-find-api": [ @@ -300,7 +353,56 @@ { "id": "winapp-before-devtools", "plugin": "winapp", - "source": "plugins/winapp (before winui-devtools)", + "source": "plugins/winapp with winapp-performance, before winui-devtools", + "skillSetHash": "93d52ca8d877", + "skills": { + "winapp-find-api": [ + "api.lookup" + ], + "winapp-find-ui": [ + "ui.samples" + ], + "winapp-frameworks": [ + "framework.guidance" + ], + "winapp-identity": [ + "identity" + ], + "winapp-manifest": [ + "msix.manifest" + ], + "winapp-maui": [ + "maui.package" + ], + "winapp-package": [ + "msix.package" + ], + "winapp-performance": [ + "winui.build" + ], + "winapp-sandbox": [ + "sandbox.run", + "ui.automate" + ], + "winapp-setup": [ + "project.setup", + "project.scaffold" + ], + "winapp-signing": [ + "msix.sign" + ], + "winapp-troubleshoot": [ + "troubleshoot" + ], + "winapp-ui-automation": [ + "ui.automate" + ] + } + }, + { + "id": "winapp-before-performance-and-devtools", + "plugin": "winapp", + "source": "plugins/winapp before winapp-performance and winui-devtools", "skillSetHash": "4a4abc8b6541", "skills": { "winapp-find-api": [ diff --git a/benchmarks/agents/lib/Benchmark.psm1 b/benchmarks/agents/lib/Benchmark.psm1 index fae19d107..a91062cdc 100644 --- a/benchmarks/agents/lib/Benchmark.psm1 +++ b/benchmarks/agents/lib/Benchmark.psm1 @@ -290,7 +290,7 @@ function Compare-PreflightSkills { function Get-WinappCommands { # Unique 'winapp []' invocations named in shell commands or answer text. param([AllowEmptyCollection()][AllowNull()][string[]]$Text) - $groups = 'cert', 'config', 'devtools', 'manifest', 'ui', 'find-api', 'target', 'store', 'node' + $groups = 'cert', 'config', 'devtools', 'manifest', 'ui', 'find-api', 'target', 'store', 'node', 'perf' # Groups whose second token can be free text record it only when it is a real subcommand: # `find-api ` is a search. `node` comes from the npm package (`npx winapp node ...`). $verbs = @{ @@ -299,7 +299,7 @@ function Get-WinappCommands { } # Top-level commands only, so prose like "winapp is" or "winapp CLI" is not counted. $known = 'az-sign', 'cert', 'config', 'create-debug-identity', 'devtools', 'create-external-catalog', 'embed-identity', 'find-api', 'find-ui', - 'get-winapp-path', 'init', 'manifest', 'new', 'node', 'package', 'pack', 'restore', 'run', 'sign', 'store', 'target', 'tool', + 'get-winapp-path', 'init', 'manifest', 'new', 'node', 'package', 'pack', 'perf', 'restore', 'run', 'sign', 'store', 'target', 'tool', 'ui', 'unregister', 'update' $found = [System.Collections.Generic.List[string]]::new() foreach ($t in $Text) { diff --git a/benchmarks/agents/tests/Benchmark.Tests.ps1 b/benchmarks/agents/tests/Benchmark.Tests.ps1 index 5c9e35c9a..2360789cf 100644 --- a/benchmarks/agents/tests/Benchmark.Tests.ps1 +++ b/benchmarks/agents/tests/Benchmark.Tests.ps1 @@ -144,6 +144,11 @@ Describe 'Get-WinappCommands' { @(Get-WinappCommands -Text @('npx winapp node create-addon --name myAddon; winapp node add-electron-debug-identity', 'winapp node something-else')) | Should -Be @('node create-addon', 'node add-electron-debug-identity', 'node') } + + It 'records perf subcommands' { + @(Get-WinappCommands -Text @('winapp perf start --app 1234 --output .\cap --json', 'winapp perf analyze .\trace --view hotspots')) | + Should -Be @('perf start', 'perf analyze') + } } Describe 'Format-PassRate' { diff --git a/benchmarks/agents/tests/Capabilities.Tests.ps1 b/benchmarks/agents/tests/Capabilities.Tests.ps1 index 1684c5bbc..28ba9ad65 100644 --- a/benchmarks/agents/tests/Capabilities.Tests.ps1 +++ b/benchmarks/agents/tests/Capabilities.Tests.ps1 @@ -187,6 +187,15 @@ Describe 'Test-ScenarioExpectations' { (Test-ScenarioExpectations -Expect $e -LoadedSkills @() -InstalledSkills $installed -WinappCommands $null -Map $map).CommandHit | Should -BeNullOrEmpty } + It 'counts a perf command toward the capability winapp-performance provides' { + $real = Read-CapabilityMap -Path (Join-Path $PSScriptRoot '..\capabilities.json') + $skills = Get-PluginSkillNames -PluginPath (Join-Path $repoRoot 'plugins\winapp') + $commands = @(Get-WinappCommands -Text 'Run winapp perf analyze .\trace --json') + $r = Test-ScenarioExpectations -Expect (New-CapExpect -primary 'winui.build') -LoadedSkills @() -InstalledSkills $skills -WinappCommands $commands -Map $real + $r.CommandHit | Should -BeTrue + (Test-ScenarioExpectations -Expect (New-CapExpect -primary 'winui.build') -LoadedSkills 'winapp-performance' -InstalledSkills $skills -WinappCommands $commands -Map $real).Status | Should -Be 'pass' + } + It 'flags skill context over budget' { $r = Test-ScenarioExpectations -Expect (New-CapExpect -primary 'msix.sign' -budget 1000) -LoadedSkills 'winapp-signing' -InstalledSkills $installed -SkillContextTokens 2500 -Map $map $r.OverBudget | Should -BeTrue diff --git a/plugins/winapp/skills/winapp-performance/SKILL.md b/plugins/winapp/skills/winapp-performance/SKILL.md index 7a29e2cb7..85dc7422e 100644 --- a/plugins/winapp/skills/winapp-performance/SKILL.md +++ b/plugins/winapp/skills/winapp-performance/SKILL.md @@ -1,6 +1,6 @@ --- name: winapp-performance -description: Investigate slow WinUI 3 layout, scrolling and virtualization using native ETW captures, timed operation trees and managed GC suspension context. Use bounded queries for runtime performance questions, not crash dumps or UI correctness. +description: Investigate slow WinUI 3 layout, scrolling and virtualization using native ETW captures, timed operation trees and managed GC suspension context. Use bounded queries for runtime performance questions, not crash analysis or UI correctness. --- ## Workflow From 3ac0a39c8b62d70b50e9b6e4bb62d91b068181ba Mon Sep 17 00:00:00 2001 From: Nikola Metulev <711864+nmetulev@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:39:14 -0700 Subject: [PATCH 2/2] Benchmark: give winapp-performance no routing credit yet Mapping it to winui.build let a perf-only load pass unrelated build and crash scenarios. No scenario measures runtime performance, so map it to no capability and stop counting perf commands toward winui.build. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 72865eb6-685a-4ede-8516-17cfe8abcc86 --- benchmarks/agents/README.md | 6 +++--- benchmarks/agents/capabilities.json | 13 ++++-------- .../agents/tests/Capabilities.Tests.ps1 | 20 ++++++++++++++----- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/benchmarks/agents/README.md b/benchmarks/agents/README.md index 08dd26074..be6254e50 100644 --- a/benchmarks/agents/README.md +++ b/benchmarks/agents/README.md @@ -331,9 +331,9 @@ forbid `winui.*`. Mapping choices that affect scores: elements). It does not provide `ui.automate`, so loading it for a click, type, read-text, or screenshot task earns no routing credit, and scenarios where it is the wrong tool forbid `winui.inspect`. Loading it in a non-WinUI scenario fails the `winui.*` forbid. -- `winapp-performance` provides `winui.build`: diagnosing a slow WinUI app is part of running and - debugging it, and `winapp perf` commands count toward it. Its scope is WinUI 3, so loading it in - a non-WinUI scenario fails the `winui.*` forbid. +- `winapp-performance` provides no capability yet: no scenario measures runtime performance, and + crediting it as `winui.build` would let it pass unrelated build and crash scenarios. Loading it + earns no routing credit, and `winapp perf` commands are recorded but count toward no capability. - Only `winui-packaging` provides `store.publish`. In the `winapp` configuration a Store scenario is `n/a`; where it is installed, a run that loads only packaging guidance scores `partial`. diff --git a/benchmarks/agents/capabilities.json b/benchmarks/agents/capabilities.json index 1ac0ea360..35d469d05 100644 --- a/benchmarks/agents/capabilities.json +++ b/benchmarks/agents/capabilities.json @@ -40,10 +40,9 @@ } }, "winui.build": { - "description": "Build, run, and debug a WinUI 3 app: build errors, startup crashes, slow layout or scrolling, and Native AOT publish.", + "description": "Build, run, and debug a WinUI 3 app: build errors, startup crashes, and Native AOT publish.", "commands": [ - "^run$", - "^perf" + "^run$" ] }, "msix.package": { @@ -276,9 +275,7 @@ "winapp-package": [ "msix.package" ], - "winapp-performance": [ - "winui.build" - ], + "winapp-performance": [], "winapp-sandbox": [ "sandbox.run", "ui.automate" @@ -377,9 +374,7 @@ "winapp-package": [ "msix.package" ], - "winapp-performance": [ - "winui.build" - ], + "winapp-performance": [], "winapp-sandbox": [ "sandbox.run", "ui.automate" diff --git a/benchmarks/agents/tests/Capabilities.Tests.ps1 b/benchmarks/agents/tests/Capabilities.Tests.ps1 index 28ba9ad65..ec531b1f8 100644 --- a/benchmarks/agents/tests/Capabilities.Tests.ps1 +++ b/benchmarks/agents/tests/Capabilities.Tests.ps1 @@ -187,13 +187,23 @@ Describe 'Test-ScenarioExpectations' { (Test-ScenarioExpectations -Expect $e -LoadedSkills @() -InstalledSkills $installed -WinappCommands $null -Map $map).CommandHit | Should -BeNullOrEmpty } - It 'counts a perf command toward the capability winapp-performance provides' { + It 'gives winapp-performance and perf commands no routing credit, in every map that has the skill' { + # No scenario measures runtime performance yet, so loading the skill must score like loading nothing. $real = Read-CapabilityMap -Path (Join-Path $PSScriptRoot '..\capabilities.json') - $skills = Get-PluginSkillNames -PluginPath (Join-Path $repoRoot 'plugins\winapp') + $winui = @(Get-PluginSkillNames -PluginPath (Join-Path $repoRoot 'plugins\winui\agent-plugin')) $commands = @(Get-WinappCommands -Text 'Run winapp perf analyze .\trace --json') - $r = Test-ScenarioExpectations -Expect (New-CapExpect -primary 'winui.build') -LoadedSkills @() -InstalledSkills $skills -WinappCommands $commands -Map $real - $r.CommandHit | Should -BeTrue - (Test-ScenarioExpectations -Expect (New-CapExpect -primary 'winui.build') -LoadedSkills 'winapp-performance' -InstalledSkills $skills -WinappCommands $commands -Map $real).Status | Should -Be 'pass' + $scenarios = @(Get-ScenarioDefinitions -ScenariosRoot (Join-Path $PSScriptRoot '..\scenarios') | Where-Object { $_.Variant -eq 'base' -and $_.Expect.Capabilities.Primary.Count }) + $maps = @($real.Maps | Where-Object { $_.Skills.ContainsKey('winapp-performance') }) + $maps.Count | Should -BeGreaterThan 0 + foreach ($m in $maps) { + $installed = @(@($m.Skills.Keys) + $winui) + foreach ($s in $scenarios) { + $none = Test-ScenarioExpectations -Expect $s.Expect -LoadedSkills @() -InstalledSkills $installed -Map $real + $r = Test-ScenarioExpectations -Expect $s.Expect -LoadedSkills 'winapp-performance' -InstalledSkills $installed -WinappCommands $commands -Map $real + $r.Status | Should -Be $none.Status -Because "$($s.Id) [$($m.Id)]" + $r.CommandHit | Should -Not -BeTrue -Because "$($s.Id) [$($m.Id)]" + } + } } It 'flags skill context over budget' {