refactor into TestAdapter and lower level TestFramework (#164)

* refactor into TestAdapter and lower level TestFramework

* it's 2024 now...

* update with first draft of test runner standalone exe

* test runner can now "list" and the start of "run"

* refactored reporter capabilities

* go back to "skipping" the "run" part for now

* with console reporter

* test reporting from standalone exe works sometimes now

* more refactor and optimizations

* fixed bug

* fix more bugs

* better prompt for test

* yaml test framework now allows bash scripts on windows using git-bash (bash.exe)

* enable `dev shell` to user process helpers class; update framework's test runner with more logger output

* remove unnecessary files

* enable better diagnostics

* more refactoring

* improved outputs

* `ait` filtering now works!!

* better usage text for `ait`

* better usage output per feedback

* updated formating

* better command line parameters and descriptions of such in usage

* stdout works properly now, and stderr now in trx file

* protect agains null when writing trx

* fixed final bugs in `ait` ... i think it's almost ready

* fix getting git bash.exe on windows, and also cache it

* bad merge

* renamed RunCommandToken to RunCommandScriptToken

.azure/pipelines/build.yaml

@@ -19,7 +19,7 @@ stages:
       name: Variables
       inputs:
         filePath: ./.azure/pipelines/scripts/set-variables.sh
-        arguments: '0.0.0-dev2023.$(Build.BuildId)'
+        arguments: '0.0.0-dev2024.$(Build.BuildId)'
       displayName: 'Set up environment variables'
 
 - stage: BuildStage

.gitignore

@@ -5,3 +5,4 @@
 **/bin/*/net7.0/*
 **/obj/*
 ideas/website/node_modules/**
+testresults/**

README.md

@@ -118,7 +118,7 @@ ai search index update --files "../../data/3-product-info/*.md" --index-name "pr
 
 ```
 AI - Azure AI CLI, Version 1.0.0
-Copyright (c) 2023 Microsoft Corporation. All Rights Reserved.
+Copyright (c) 2024 Microsoft Corporation. All Rights Reserved.
 
 This PUBLIC PREVIEW version may change at any time.
 See: https://aka.ms/azure-ai-cli-public-preview

ai-cli.sln

@@ -17,6 +17,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "template_extension", "src\e
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YamlTestAdapter", "tests\testadapter\YamlTestAdapter.csproj", "{7C3F1355-B679-487D-904D-7E5FEBA9E75C}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YamlTestFramework", "tests\testframework\YamlTestFramework.csproj", "{B0B3437F-1828-4A13-866F-1CF7C924015E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YamlTestRunner", "tests\testrunner\YamlTestRunner.csproj", "{39876475-2D98-40CF-8B08-CD423A5EB4E8}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{C8AFF891-D6AA-4B8F-BC21-10404DF4B355}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -47,6 +53,14 @@ Global
 		{7C3F1355-B679-487D-904D-7E5FEBA9E75C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{7C3F1355-B679-487D-904D-7E5FEBA9E75C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7C3F1355-B679-487D-904D-7E5FEBA9E75C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B0B3437F-1828-4A13-866F-1CF7C924015E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B0B3437F-1828-4A13-866F-1CF7C924015E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B0B3437F-1828-4A13-866F-1CF7C924015E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B0B3437F-1828-4A13-866F-1CF7C924015E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{39876475-2D98-40CF-8B08-CD423A5EB4E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{39876475-2D98-40CF-8B08-CD423A5EB4E8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{39876475-2D98-40CF-8B08-CD423A5EB4E8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{39876475-2D98-40CF-8B08-CD423A5EB4E8}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -55,6 +69,9 @@ Global
 		{272E0B1B-6C05-428E-BF64-E30B1E5F603A} = {644B75F1-C768-4DB3-BAF2-C69A1F36DD28}
 		{7BD6EF67-BA75-478D-9721-C1B2AB6DE3FF} = {644B75F1-C768-4DB3-BAF2-C69A1F36DD28}
 		{023B4F9C-E2B3-4CCD-A993-87E337C16EDE} = {644B75F1-C768-4DB3-BAF2-C69A1F36DD28}
+		{7C3F1355-B679-487D-904D-7E5FEBA9E75C} = {C8AFF891-D6AA-4B8F-BC21-10404DF4B355}
+		{B0B3437F-1828-4A13-866F-1CF7C924015E} = {C8AFF891-D6AA-4B8F-BC21-10404DF4B355}
+		{39876475-2D98-40CF-8B08-CD423A5EB4E8} = {C8AFF891-D6AA-4B8F-BC21-10404DF4B355}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {002655B1-E1E1-4F2A-8D53-C9CD55136AE2}

src/ai/Program_AI.cs

@@ -85,11 +85,11 @@ public class AiProgramData : IProgramData
         #endregion
 
         #region help command data
-        public string HelpCommandTokens => "wizard;dev;init;config;chat;flow;speech;vision;language;search;service;tool;samples;code;eval;run";
+        public string HelpCommandTokens => "wizard;dev;init;config;chat;flow;speech;vision;language;search;service;tool;samples;eval;run";
         #endregion
 
         #region config command data
-        public string ConfigScopeTokens => $"wizard;dev;init;chat;flow;speech;vision;language;search;service;tool;samples;code;eval;run;*";
+        public string ConfigScopeTokens => $"wizard;dev;init;chat;flow;speech;vision;language;search;service;tool;samples;eval;run;*";
         #endregion
 
         #region zip option data
@@ -150,8 +150,6 @@ public bool DispatchRunCommand(ICommandValues values)
                 "search" => (new SearchCommand(values)).RunCommand(),
                 "service" => (new ServiceCommand(values)).RunCommand(),
                 "tool" => (new ToolCommand(values)).RunCommand(),
-                "samples" => (new SamplesCommand(values)).RunCommand(),
-                "code" => (new CodeCommand(values)).RunCommand(),
                 "eval" => (new EvalCommand(values)).RunCommand(),
                 "wizard" => (new ScenarioWizardCommand(values)).RunCommand(),
                 "dev" => (new DevCommand(values)).RunCommand(),
@@ -179,8 +177,6 @@ public bool DispatchParseCommand(INamedValueTokens tokens, ICommandValues values
                 "search" => SearchCommandParser.ParseCommand(tokens, values),
                 "service" => ServiceCommandParser.ParseCommand(tokens, values),
                 "tool" => ToolCommandParser.ParseCommand(tokens, values),
-                "samples" => SamplesCommandParser.ParseCommand(tokens, values),
-                "code" => CodeCommandParser.ParseCommand(tokens, values),
                 "wizard" => ScenarioWizardCommandParser.ParseCommand(tokens, values),
                 "dev" => DevCommandParser.ParseCommand(tokens, values),
                 "run" => RunJobCommandParser.ParseCommand(tokens, values),
@@ -204,9 +200,6 @@ public bool DispatchParseCommandValues(INamedValueTokens tokens, ICommandValues
                 "search" => SearchCommandParser.ParseCommandValues(tokens, values),
                 "service" => ServiceCommandParser.ParseCommandValues(tokens, values),
                 "tool" => ToolCommandParser.ParseCommandValues(tokens, values),
-                "samples" => SamplesCommandParser.ParseCommandValues(tokens, values),
-                "code" => CodeCommandParser.ParseCommandValues(tokens, values),
-                "complete" => CompleteCommandParser.ParseCommandValues(tokens, values),
                 "wizard" => ScenarioWizardCommandParser.ParseCommandValues(tokens, values),
                 "dev" => DevCommandParser.ParseCommandValues(tokens, values),
                 "run" => RunJobCommandParser.ParseCommandValues(tokens, values),