diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs index a238267e72..14823a4e32 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs @@ -39,7 +39,7 @@ public void SendMessage(TestMessageLevel testMessageLevel, string message) /// private readonly IDictionary _sessionParameters; private readonly IEnvironment _environment; - private readonly Func _taskFactory; + private readonly Func, Task> _taskFactory; /// /// Specifies whether the test run is canceled or not. @@ -51,7 +51,7 @@ public TestExecutionManager() { } - internal TestExecutionManager(IEnvironment environment, Func? taskFactory = null) + internal TestExecutionManager(IEnvironment environment, Func, Task>? taskFactory = null) { _testMethodFilter = new TestMethodFilter(); _sessionParameters = new Dictionary(); @@ -59,7 +59,7 @@ internal TestExecutionManager(IEnvironment environment, Func? task _taskFactory = taskFactory ?? DefaultFactoryAsync; } - private static Task DefaultFactoryAsync(Action action) + private static Task DefaultFactoryAsync(Func taskGetter) { if (MSTestSettings.RunConfigurationSettings.ExecutionApartmentState == ApartmentState.STA && RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -69,7 +69,9 @@ private static Task DefaultFactoryAsync(Action action) { try { - action(); + // This is best we can do to execute in STA thread. + Task task = taskGetter(); + task.GetAwaiter().GetResult(); tcs.SetResult(0); } catch (Exception ex) @@ -84,7 +86,8 @@ private static Task DefaultFactoryAsync(Action action) } else { - return Task.Run(action); + // NOTE: If you replace this with `return taskGetter()`, you will break parallel tests. + return Task.Run(taskGetter); } } @@ -121,7 +124,9 @@ public void RunTests(IEnumerable tests, IRunContext? runContext, IFram CacheSessionParameters(runContext, frameworkHandle); // Execute the tests - ExecuteTests(tests, runContext, frameworkHandle, isDeploymentDone); + // This is a public API, so we can't change it to be async. + // Consider not using this API internally, and introduce an async version, and mark this as obsolete. + ExecuteTestsAsync(tests, runContext, frameworkHandle, isDeploymentDone).GetAwaiter().GetResult(); if (!_hasAnyTestFailed) { @@ -159,7 +164,9 @@ public void RunTests(IEnumerable sources, IRunContext? runContext, IFram CacheSessionParameters(runContext, frameworkHandle); // Run tests. - ExecuteTests(tests, runContext, frameworkHandle, isDeploymentDone); + // This is a public API, so we can't change it to be async. + // Consider not using this API internally, and introduce an async version, and mark this as obsolete. + ExecuteTestsAsync(tests, runContext, frameworkHandle, isDeploymentDone).GetAwaiter().GetResult(); if (!_hasAnyTestFailed) { @@ -174,7 +181,7 @@ public void RunTests(IEnumerable sources, IRunContext? runContext, IFram /// The run context. /// Handle to record test start/end/results. /// Indicates if deployment is done. - internal virtual void ExecuteTests(IEnumerable tests, IRunContext? runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) + internal virtual async Task ExecuteTestsAsync(IEnumerable tests, IRunContext? runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) { var testsBySource = from test in tests group test by test.Source into testGroup @@ -183,7 +190,7 @@ group test by test.Source into testGroup foreach (var group in testsBySource) { _testRunCancellationToken?.ThrowIfCancellationRequested(); - ExecuteTestsInSource(group.Tests, runContext, frameworkHandle, group.Source, isDeploymentDone); + await ExecuteTestsInSourceAsync(group.Tests, runContext, frameworkHandle, group.Source, isDeploymentDone); } } @@ -257,7 +264,7 @@ private static bool MatchTestFilter(ITestCaseFilterExpression? filterExpression, /// Handle to record test start/end/results. /// The test container for the tests. /// Indicates if deployment is done. - private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone) + private async Task ExecuteTestsInSourceAsync(IEnumerable tests, IRunContext? runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone) { DebugEx.Assert(!StringEx.IsNullOrEmpty(source), "Source cannot be empty"); @@ -267,7 +274,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC } using MSTestAdapter.PlatformServices.Interface.ITestSourceHost isolationHost = PlatformServiceProvider.Instance.CreateTestSourceHost(source, runContext?.RunSettings, frameworkHandle); - + bool usesAppDomains = isolationHost is MSTestAdapter.PlatformServices.TestSourceHost { UsesAppDomain: true }; PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Created unit-test runner {0}", source); // Default test set is filtered tests based on user provided filter criteria @@ -363,7 +370,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC { _testRunCancellationToken?.ThrowIfCancellationRequested(); - tasks.Add(_taskFactory(() => + tasks.Add(_taskFactory(async () => { try { @@ -373,7 +380,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC if (queue.TryDequeue(out IEnumerable? testSet)) { - ExecuteTestsWithTestRunner(testSet, frameworkHandle, source, sourceLevelParameters, testRunner); + await ExecuteTestsWithTestRunnerAsync(testSet, frameworkHandle, source, sourceLevelParameters, testRunner, usesAppDomains); } } } @@ -385,7 +392,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC try { - Task.WaitAll(tasks.ToArray()); + await Task.WhenAll(tasks); } catch (Exception ex) { @@ -399,12 +406,12 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC // Queue the non parallel set if (nonParallelizableTestSet != null) { - ExecuteTestsWithTestRunner(nonParallelizableTestSet, frameworkHandle, source, sourceLevelParameters, testRunner); + await ExecuteTestsWithTestRunnerAsync(nonParallelizableTestSet, frameworkHandle, source, sourceLevelParameters, testRunner, usesAppDomains); } } else { - ExecuteTestsWithTestRunner(testsToRun, frameworkHandle, source, sourceLevelParameters, testRunner); + await ExecuteTestsWithTestRunnerAsync(testsToRun, frameworkHandle, source, sourceLevelParameters, testRunner, usesAppDomains); } if (PlatformServiceProvider.Instance.IsGracefulStopRequested) @@ -415,12 +422,13 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed tests belonging to source {0}", source); } - private void ExecuteTestsWithTestRunner( + private async Task ExecuteTestsWithTestRunnerAsync( IEnumerable tests, ITestExecutionRecorder testExecutionRecorder, string source, IDictionary sourceLevelParameters, - UnitTestRunner testRunner) + UnitTestRunner testRunner, + bool usesAppDomains) { bool hasAnyRunnableTests = false; var fixtureTests = new List(); @@ -429,7 +437,11 @@ private void ExecuteTestsWithTestRunner( ? tests.OrderBy(t => t.GetManagedType()).ThenBy(t => t.GetManagedMethod()) : tests; - var remotingMessageLogger = new RemotingMessageLogger(testExecutionRecorder); + // If testRunner is in a different AppDomain, we cannot pass the testExecutionRecorder directly. + // Instead, we pass a proxy (remoting object) that is marshallable by ref. + IMessageLogger remotingMessageLogger = usesAppDomains + ? new RemotingMessageLogger(testExecutionRecorder) + : testExecutionRecorder; foreach (TestCase currentTest in orderedTests) { @@ -460,9 +472,18 @@ private void ExecuteTestsWithTestRunner( IDictionary tcmProperties = TcmTestPropertiesProvider.GetTcmProperties(currentTest); Dictionary testContextProperties = GetTestContextProperties(tcmProperties, sourceLevelParameters); - // testRunner could be in a different AppDomain. We cannot pass the testExecutionRecorder directly. - // Instead, we pass a proxy (remoting object) that is marshallable by ref. - UnitTestResult[] unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testContextProperties, remotingMessageLogger); + UnitTestResult[] unitTestResult; + if (usesAppDomains) + { +#pragma warning disable VSTHRD103 // Call async methods when in an async method - We cannot do right now because we are crossing app domains. + // TODO: When app domains support is dropped, we can finally always be calling the async version. + unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testContextProperties, remotingMessageLogger); +#pragma warning restore VSTHRD103 // Call async methods when in an async method + } + else + { + unitTestResult = await testRunner.RunSingleTestAsync(unitTestElement.TestMethod, testContextProperties, remotingMessageLogger); + } PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed test {0}", unitTestElement.TestMethod.Name); diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs index 3b61f1aae0..4085813c23 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs @@ -46,6 +46,7 @@ internal TestMethodInfo( Parent = parent; TestMethodOptions = testMethodOptions; ExpectedException = ResolveExpectedException(); + RetryAttribute = GetRetryAttribute(); } /// @@ -92,6 +93,8 @@ internal TestMethodInfo( internal ExpectedExceptionBaseAttribute? ExpectedException { get; set; /*set for testing only*/ } + internal RetryBaseAttribute? RetryAttribute { get; } + public Attribute[]? GetAllAttributes(bool inherit) => ReflectHelper.Instance.GetDerivedAttributes(TestMethod, inherit).ToArray(); public TAttributeType[] GetAttributes(bool inherit) @@ -249,17 +252,52 @@ public virtual TestResult Invoke(object?[]? arguments) // See https://github.com/microsoft/testfx/issues/4331 if (expectedExceptions.Count() > 1) { - string errorMessage = string.Format( - CultureInfo.CurrentCulture, - Resource.UTA_MultipleExpectedExceptionsOnTestMethod, - Parent.ClassType.FullName, - TestMethod.Name); - throw new TypeInspectionException(errorMessage); + ThrowMultipleAttributesException(nameof(ExpectedExceptionBaseAttribute)); } return expectedExceptions.FirstOrDefault(); } + /// + /// Gets the number of retries this test method should make in case of failure. + /// + /// + /// The number of retries, which is always greater than or equal to 1. + /// If RetryAttribute is not present, returns 1. + /// + private RetryBaseAttribute? GetRetryAttribute() + { + IEnumerable attributes = ReflectHelper.Instance.GetDerivedAttributes(TestMethod, inherit: true); + using IEnumerator enumerator = attributes.GetEnumerator(); + if (!enumerator.MoveNext()) + { + return null; + } + + RetryBaseAttribute attribute = enumerator.Current; + + if (enumerator.MoveNext()) + { + ThrowMultipleAttributesException(nameof(RetryBaseAttribute)); + } + + return attribute; + } + + [DoesNotReturn] + private void ThrowMultipleAttributesException(string attributeName) + { + // Note: even if the given attribute has AllowMultiple = false, we can + // still reach here if a derived attribute authored by the user re-defines AttributeUsage + string errorMessage = string.Format( + CultureInfo.CurrentCulture, + Resource.UTA_MultipleAttributesOnTestMethod, + Parent.ClassType.FullName, + TestMethod.Name, + attributeName); + throw new TypeInspectionException(errorMessage); + } + /// /// Execute test without timeout. /// diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs index cb736eab47..249301cd2d 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs @@ -173,6 +173,7 @@ internal List RunTestMethod() { if (_test.TestDataSourceIgnoreMessage is not null) { + _testContext.SetOutcome(UTF.UnitTestOutcome.Ignored); return [new() { Outcome = UTF.UnitTestOutcome.Ignored, IgnoreReason = _test.TestDataSourceIgnoreMessage }]; } diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index 563085f288..7a969b0dfb 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -6,6 +6,7 @@ using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -122,13 +123,19 @@ internal FixtureTestResult GetFixtureTestResult(TestMethod testMethod, string fi static UnitTestOutcome GetOutcome(Exception? exception) => exception == null ? UnitTestOutcome.Passed : UnitTestOutcome.Failed; } + // Task cannot cross app domains. + // For now, TestExecutionManager will call this sync method which is hacky. + // If we removed AppDomains in v4, we should use the async method and remove this one. + internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary testContextProperties, IMessageLogger messageLogger) + => RunSingleTestAsync(testMethod, testContextProperties, messageLogger).GetAwaiter().GetResult(); + /// /// Runs a single test. /// /// The test Method. /// The test context properties. /// The . - internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary testContextProperties, IMessageLogger messageLogger) + internal async Task RunSingleTestAsync(TestMethod testMethod, IDictionary testContextProperties, IMessageLogger messageLogger) { Guard.NotNull(testMethod); @@ -179,8 +186,23 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary firstRunResult = testMethodRunner.Execute(classInitializeResult.StandardOut!, classInitializeResult.StandardError!, classInitializeResult.DebugTrace!, classInitializeResult.TestContextMessages!); + result = firstRunResult.ToUnitTestResults(); + if (retryAttribute is not null && !RetryBaseAttribute.IsAcceptableResultForRetry(firstRunResult)) + { + RetryResult retryResult = await retryAttribute.ExecuteAsync( + new RetryContext( + () => Task.FromResult( + testMethodRunner.Execute( + classInitializeResult.StandardOut!, + classInitializeResult.StandardError!, + classInitializeResult.DebugTrace!, + classInitializeResult.TestContextMessages!).ToArray()))); + + result = retryResult.TryGetLast()?.ToUnitTestResults() ?? throw ApplicationStateGuard.Unreachable(); + } } } } diff --git a/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs b/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs index 6f84f352db..8daf3f194b 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs +++ b/src/Adapter/MSTest.TestAdapter/Resources/Resource.Designer.cs @@ -777,11 +777,11 @@ internal static string UTA_MethodDoesNotExists { } /// - /// Looks up a localized string similar to The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed.. + /// Looks up a localized string similar to The test method {0}.{1} has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed.. /// - internal static string UTA_MultipleExpectedExceptionsOnTestMethod { + internal static string UTA_MultipleAttributesOnTestMethod { get { - return ResourceManager.GetString("UTA_MultipleExpectedExceptionsOnTestMethod", resourceCulture); + return ResourceManager.GetString("UTA_MultipleAttributesOnTestMethod", resourceCulture); } } diff --git a/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx b/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx index 67131b4985..2d8bfb92aa 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx +++ b/src/Adapter/MSTest.TestAdapter/Resources/Resource.resx @@ -268,8 +268,8 @@ Error: {1} The ExpectedException attribute defined on test method {0}.{1} threw an exception during construction. {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. @@ -415,4 +415,4 @@ but received {4} argument(s), with types '{5}'. Found two conflicting types for generic parameter '{0}'. The conflicting types are '{1}' and '{2}'. - \ No newline at end of file + diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf index fde9e9782f..d873f52521 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.cs.xlf @@ -171,6 +171,11 @@ byl však přijat tento počet argumentů: {4} s typy {5}. Nepodařilo se vytvořit instanci třídy {0}. Chyba: {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. Nelze najít platný konstruktor pro testovací třídu {0}. Platné konstruktory jsou public a buď bez parametrů, nebo s jedním parametrem typu TestContext. @@ -364,11 +369,6 @@ Chyba: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - Testovací metoda {0}.{1} má definovaných více atributů odvozených od atributu ExpectedExceptionBaseAttribute. Povolený je jenom jeden takový atribut. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Upozornění: Adaptér MSTest V2 nepodporuje soubor testsettings ani vsmdi. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf index 6f057f182b..753ad47e69 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.de.xlf @@ -171,6 +171,11 @@ aber empfing {4} Argument(e) mit den Typen „{5}“. Es kann keine Instanz der Klasse '{0}' erstellt werden. Fehler: {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. Es wurde kein gültiger Konstruktor für die Testklasse "{0}" gefunden. Gültige Konstruktoren sind "public" und entweder parameterlos oder mit einem Parameter vom Typ "TestContext". @@ -364,11 +369,6 @@ Fehler: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - Für die Testmethode "{0}.{1}" sind mehrere Attribute definiert, die von ExpectedExceptionBaseAttribute abgeleitet werden. Nur ein einziges solches Attribut ist zulässig. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Warnung: Eine TESTSETTINGS-Datei oder eine VSMDI-Datei wird vom MSTest-V2-Adapter nicht unterstützt. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf index 1642c79ef6..a6c7cce79b 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.es.xlf @@ -171,6 +171,11 @@ pero recibió {4} argumento(s), con los tipos "{5}". No se puede crear una instancia de la clase {0}. Error: {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. No se encuentra un constructor válido para la clase de prueba '{0}'. Los constructores válidos son 'public' y sin parámetros o con un parámetro de tipo 'TestContext'. @@ -364,11 +369,6 @@ Error: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - El método de prueba {0}.{1} tiene definidos varios atributos derivados de ExpectedExceptionBaseAttribute. Solo se permite uno de estos atributos. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Advertencia: No se admite un archivo testsettings o vsmdi con el adaptador de MSTest V2. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf index c9dcb474d5..83d5e38a64 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.fr.xlf @@ -171,6 +171,11 @@ mais a reçu {4} argument(s), avec les types « {5} ». Impossible de créer une instance de la classe {0}. Erreur : {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. Impossible de trouver un constructeur valide pour la classe de test « {0} ». Les constructeurs valides sont « publics » et sans paramètre ou avec un paramètre de type « TestContext ». @@ -364,11 +369,6 @@ Erreur : {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - Plusieurs attributs dérivés de ExpectedExceptionBaseAttribute sont définis dans la méthode de test {0}.{1}. Un seul attribut de ce type est autorisé. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Avertissement : L'adaptateur MSTest V2 ne prend pas en charge les fichiers testsettings ou vsmdi. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf index d8b6938927..05fe1e9741 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.it.xlf @@ -171,6 +171,11 @@ ma ha ricevuto {4} argomenti, con tipi "{5}". Non è possibile creare un'istanza della classe {0}. Errore: {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. Impossibile trovare un costruttore valido per la classe di test '{0}'. I costruttori validi sono 'public' e senza parametri o con un parametro di tipo 'TestContext'. @@ -364,11 +369,6 @@ Errore: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - Il metodo di test {0}.{1} contiene più attributi derivati dall'elemento ExpectedExceptionBaseAttribute definito per esso. È consentito solo uno di tali attributi. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Avviso: con l'adattatore MSTest V2 non è possibile usare un file testsettings o un file vsmdi. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf index 4af55974bf..3c76e4c7bb 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ja.xlf @@ -172,6 +172,11 @@ but received {4} argument(s), with types '{5}'. クラス {0} のインスタンスを作成できません。エラー: {1}。 + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. テスト クラス '{0}' の有効なコンストラクターが見つかりません。有効なコンストラクターは、'public' で、パラメーターがないもの、または 'TestContext' 型のパラメーター 1 個を取るものです。 @@ -365,11 +370,6 @@ Error: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - テスト メソッド {0}.{1} には、ExpectedExceptionBaseAttribute から派生した属性が複数定義されています。このような属性は 1 つしか許可されません。 - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. 警告: testsettings ファイル、vsmdi ファイルは MSTest V2 アダプターではサポートされていません。 diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf index cb7a5de539..b4e66cae85 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ko.xlf @@ -171,6 +171,11 @@ but received {4} argument(s), with types '{5}'. {0} 클래스의 인스턴스를 만들 수 없습니다. 오류: {1} + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. 테스트 클래스 '{0}'에 대한 유효한 생성자를 찾을 수 없습니다. 유효한 생성자는 'public'이며 매개 변수가 없거나 'TestContext' 유형의 매개 변수가 하나 있습니다. @@ -364,11 +369,6 @@ Error: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - 테스트 메서드 {0}.{1}에는 정의되어 있는 ExpectedExceptionBaseAttribute에서 파생된 특성이 여러 개 있습니다. 이러한 특성은 하나만 허용됩니다. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. 경고: MSTest V2 어댑터에서는 testsettings 파일 또는 vsmdi 파일이 지원되지 않습니다. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf index 445457e66a..717ca2a562 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pl.xlf @@ -171,6 +171,11 @@ ale liczba odebranych argumentów to {4} z typami „{5}”. Nie można utworzyć wystąpienia klasy {0}. Błąd: {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. Nie można odnaleźć prawidłowego konstruktora dla klasy testowej „{0}”. Prawidłowe konstruktory są „publiczne” i albo bez parametrów, albo z jednym parametrem typu „TestContext”. @@ -364,11 +369,6 @@ Błąd: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - Metoda testowa {0}.{1} ma wiele zdefiniowanych dla niej atrybutów pochodzących od atrybutu ExpectedExceptionBaseAttribute. Dozwolony jest tylko jeden taki atrybut. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Ostrzeżenie: Plik testsettings lub plik vsmdi nie jest obsługiwany przez adapter MSTest w wersji 2. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf index 6c0cc9fdd2..21994f692a 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.pt-BR.xlf @@ -171,6 +171,11 @@ mas {4} argumentos recebidos, com tipos '{5}'. Não é possível criar instância da classe {0}. Erro: {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. Não é possível localizar um construtor válido para a classe de teste '{0}'. Construtores válidos são 'public' e sem parâmetros ou com um parâmetro do tipo 'TestContext'. @@ -364,11 +369,6 @@ Erro: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - O método de teste {0}.{1} tem vários atributos derivados de ExpectedExceptionBaseAttribute definidos nele. Somente um atributo é permitido. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Aviso: um arquivo testsettings ou um arquivo vsmdi não tem suporte no MSTest V2 Adapter. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf index a77f9d4e60..0cb108ae0f 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.ru.xlf @@ -171,6 +171,11 @@ but received {4} argument(s), with types '{5}'. Не удалось создать экземпляр класса {0}. Ошибка: {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. Не удается найти допустимый конструктор для тестового класса "{0}". Допустимые конструкторы : "public" и конструкторы без параметров или с одним параметром типа "TestContext". @@ -364,11 +369,6 @@ Error: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - В методе теста {0}.{1} определено несколько атрибутов, производных от ExpectedExceptionBaseAttribute, заданного в нем. Допускается только один такой атрибут. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Внимание! Адаптер MSTest версии 2 не поддерживает файл TESTSETTINGS или VSMDI. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf index c089c801d9..52c9f0dd13 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.tr.xlf @@ -171,6 +171,11 @@ ancak, '{5}' türüyle {4} argüman aldı. {0} sınıfının örneği oluşturulamıyor. Hata: {1}. + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. '{0}' test sınıfı için geçerli bir oluşturucu bulunamıyor. Geçerli oluşturucular 'public' ve parametresiz veya 'TestContext' türünde tek bir parametre içeriyor. @@ -364,11 +369,6 @@ Hata: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - {0}.{1} test yöntemi, üzerinde tanımlanan ExpectedExceptionBaseAttribute öğesinden türetilmiş birden fazla öznitelik içeriyor. Bu türde yalnızca bir tane özniteliğe izin verilir. - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. Uyarı : MSTest V2 Adapter ile bir testsettings dosyası veya bir vsmdi dosyası desteklenmez. diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf index cd03d76e58..b8f32397fb 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hans.xlf @@ -171,6 +171,11 @@ but received {4} argument(s), with types '{5}'. 无法创建类 {0} 的实例。错误: {1}。 + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. 找不到测试类“{0}”的有效构造函数。有效的构造函数为 “public”，但该构造函数无参数或具有一个类型为 “TestContext” 的参数。 @@ -364,11 +369,6 @@ Error: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - 测试方法 {0}.{1} 具有多个在该方法上定义的 ExpectedExceptionBaseAttribute 的派生属性。仅允许一个此类属性。 - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. 警告: MSTest V2 适配器不支持 testsettings 文件或 vsmdi 文件。 diff --git a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf index 0333e3a8f8..14e6fa432b 100644 --- a/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf +++ b/src/Adapter/MSTest.TestAdapter/Resources/xlf/Resource.zh-Hant.xlf @@ -171,6 +171,11 @@ but received {4} argument(s), with types '{5}'. 無法建立類別 {0} 的執行個體。錯誤: {1}。 + + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + The test method '{0}.{1}' has multiple attributes derived from '{2}' defined on it. Only one such attribute is allowed. + + Cannot find a valid constructor for test class '{0}'. Valid constructors are 'public' and either parameterless or with one parameter of type 'TestContext'. 找不到測試類別 '{0}' 的有效建構函式。有效的建構函式為 'public' 且無參數或具有一個類型為 'TestContext' 的參數。 @@ -364,11 +369,6 @@ Error: {1} {2} - - The test method {0}.{1} has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed. - 測試方法 {0}。{1} 上定義了多個衍生自 ExpectedExceptionBaseAttribute 的屬性。只允許一個此類屬性。 - - Warning : A testsettings file or a vsmdi file is not supported with the MSTest V2 Adapter. 警告: MSTest V2 配接器不支援 testsettings 檔案 vsmdi 檔案。 diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Services/TestSourceHost.cs b/src/Adapter/MSTestAdapter.PlatformServices/Services/TestSourceHost.cs index 441b3c7d5d..ee9deb39cb 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Services/TestSourceHost.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Services/TestSourceHost.cs @@ -104,6 +104,14 @@ internal TestSourceHost(string sourceFileName, IRunSettings? runSettings, IFrame internal AppDomain? AppDomain { get; private set; } #endif +#pragma warning disable CA1822 // Mark members as static - accesses instance data under .NET Framework + internal bool UsesAppDomain => +#if NETFRAMEWORK + !_isAppDomainCreationDisabled; +#else + false; +#endif + /// /// Setup the isolation host. /// diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/DelayBackoffType.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/DelayBackoffType.cs new file mode 100644 index 0000000000..416d86b363 --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/DelayBackoffType.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Specifies a backoff type for the delay between retries. +/// +public enum DelayBackoffType +{ + /// + /// Specifies a constant backoff type. Meaning the delay between retries is constant. + /// + Constant, + + /// + /// Specifies an exponential backoff type. + /// The delay is calculated as the base delay * 2^(n-1) where n is the retry attempt. + /// For example, if the base delay is 1000ms, the delays will be 1000ms, 2000ms, 4000ms, 8000ms, etc. + /// + Exponential, +} diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/RetryAttribute.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryAttribute.cs new file mode 100644 index 0000000000..f57242da16 --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryAttribute.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// This attribute is used to set a retry count on a test method in case of failure. +/// +[AttributeUsage(AttributeTargets.Method, Inherited = false, AllowMultiple = false)] +public sealed class RetryAttribute : RetryBaseAttribute +{ + /// + /// Initializes a new instance of the class with the given number of max retries. + /// + /// The maximum number of retry attempts. This must be greater than or equal to 1. + public RetryAttribute(int maxRetryAttempts) + { + if (maxRetryAttempts < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxRetryAttempts)); + } + + MaxRetryAttempts = maxRetryAttempts; + } + + /// + /// Gets the number of retries that the test should make in case of failures. + /// Note that before RetryAttribute is considered, the test was already executed once. + /// This property determines the max number of retries after the first normal run. + /// + public int MaxRetryAttempts { get; } + + /// + /// Gets or sets the delay, in milliseconds, between retries. + /// This delay is also applied after the first run and before the first retry attempt. + /// + public int MillisecondsDelayBetweenRetries { get; set; } + + public DelayBackoffType BackoffType + { + get => field; + set + { + if (value is < DelayBackoffType.Constant or > DelayBackoffType.Exponential) + { + throw new ArgumentOutOfRangeException(nameof(value)); + } + + field = value; + } + } + + /// + /// Retries the test method times in case of failure. + /// Note that a first run of the method was already executed and failed before this method is called. + /// + /// An object to encapsulate the state needed for retry execution. + /// + /// Returns a object that contains the results of all attempts. Only + /// the last added element is used to determine the test outcome. + /// The other results are currently not used, but may be used in the future for tooling to show the + /// state of the failed attempts. + /// + protected internal override async Task ExecuteAsync(RetryContext retryContext) + { + var result = new RetryResult(); + int currentDelay = MillisecondsDelayBetweenRetries; + for (int i = 0; i < MaxRetryAttempts; i++) + { + // The caller already executed the test once. So we need to do the delay here. + await Task.Delay(currentDelay); + if (BackoffType == DelayBackoffType.Exponential) + { + currentDelay *= 2; + } + + TestResult[] testResults = await retryContext.ExecuteTaskGetter(); + result.AddResult(testResults); + if (IsAcceptableResultForRetry(testResults)) + { + break; + } + } + + return result; + } +} diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/RetryBaseAttribute.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryBaseAttribute.cs new file mode 100644 index 0000000000..667249f437 --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryBaseAttribute.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// An abstract attribute that controls retrying a test method if it failed. It's up to the derived classes to +/// define how the retry is done. +/// +[AttributeUsage(AttributeTargets.Method, Inherited = false, AllowMultiple = false)] +public abstract class RetryBaseAttribute : Attribute +{ + /// + /// Retries the test method. The details of how retry is done is left to the derived classes. + /// Note that a first run of the method was already executed and failed before this method is called. + /// + /// An object to encapsulate the state needed for retry execution. + /// + /// Returns a object that contains the results of all attempts. Only + /// the last added element is used to determine the test outcome. + /// The other results are currently not used, but may be used in the future for tooling to show the + /// state of the failed attempts. + /// + protected internal abstract Task ExecuteAsync(RetryContext retryContext); + + internal static bool IsAcceptableResultForRetry(IEnumerable results) + { + foreach (TestResult result in results) + { + UnitTestOutcome outcome = result.Outcome; + if (outcome is UnitTestOutcome.Failed or UnitTestOutcome.Timeout) + { + return false; + } + } + + return true; + } +} diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/RetryContext.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryContext.cs new file mode 100644 index 0000000000..8d8be78473 --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryContext.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +public readonly struct RetryContext +{ + internal RetryContext(Func> executeTaskGetter) + => ExecuteTaskGetter = executeTaskGetter; + + public Func> ExecuteTaskGetter { get; } +} diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/RetryResult.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryResult.cs new file mode 100644 index 0000000000..7d1337450f --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryResult.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +public sealed class RetryResult +{ + private readonly List _testResults = new(); + + public void AddResult(TestResult[] testResults) + => _testResults.Add(testResults); + + internal TestResult[]? TryGetLast() + => _testResults.Count > 0 ? _testResults[_testResults.Count - 1] : null; +} diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index 01ad108d22..f3bbda5e6a 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -1,4 +1,5 @@ #nullable enable +abstract Microsoft.VisualStudio.TestTools.UnitTesting.RetryBaseAttribute.ExecuteAsync(Microsoft.VisualStudio.TestTools.UnitTesting.RetryContext retryContext) -> System.Threading.Tasks.Task! Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AssertAreEqualInterpolatedStringHandler Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AssertAreEqualInterpolatedStringHandler.AppendFormatted(object? value, int alignment = 0, string? format = null) -> void Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AssertAreEqualInterpolatedStringHandler.AppendFormatted(string? value) -> void @@ -190,6 +191,9 @@ Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AssertThrowsExactlyInterpola Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AssertThrowsExactlyInterpolatedStringHandler.AssertThrowsExactlyInterpolatedStringHandler(int literalLength, int formattedCount, System.Action! action, out bool shouldAppend) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IgnoreMessage.get -> string? Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IgnoreMessage.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType +Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType.Constant = 0 -> Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType +Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType.Exponential = 1 -> Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType) -> void @@ -202,6 +206,21 @@ Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.AutoDetect = Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.IgnoreMessage.get -> string? Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.IgnoreMessage.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.BackoffType.get -> Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.BackoffType.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.MaxRetryAttempts.get -> int +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.MillisecondsDelayBetweenRetries.get -> int +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.MillisecondsDelayBetweenRetries.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.RetryAttribute(int maxRetryAttempts) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryBaseAttribute +Microsoft.VisualStudio.TestTools.UnitTesting.RetryBaseAttribute.RetryBaseAttribute() -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryContext +Microsoft.VisualStudio.TestTools.UnitTesting.RetryContext.ExecuteTaskGetter.get -> System.Func!>! +Microsoft.VisualStudio.TestTools.UnitTesting.RetryContext.RetryContext() -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryResult +Microsoft.VisualStudio.TestTools.UnitTesting.RetryResult.AddResult(Microsoft.VisualStudio.TestTools.UnitTesting.TestResult![]! testResults) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryResult.RetryResult() -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.IgnoreMessage.get -> string? Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.IgnoreMessage.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.IgnoreMessage.get -> string? diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs new file mode 100644 index 0000000000..2b307abb47 --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class RetryTests : AcceptanceTestBase +{ + [TestMethod] + public async Task BasicRetryScenarioTest() + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, TargetFrameworks.NetCurrent); + TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings"); + + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + testHostResult.AssertOutputContains(""" + TestMethod1 executed 1 time. + TestMethod2 executed 2 times. + TestMethod3 executed 3 times. + TestMethod4 executed 4 times. + TestMethod5 executed 4 times. + """); + + testHostResult.AssertOutputContains("failed TestMethod5"); + testHostResult.AssertOutputContains("Assert.Fail failed. Failing TestMethod4. Attempts: 4"); + testHostResult.AssertOutputContainsSummary(failed: 1, passed: 4, skipped: 0); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) + { + public const string ProjectName = "RetryTests"; + + public string ProjectPath => GetAssetPath(ProjectName); + + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() + { + yield return (ProjectName, ProjectName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + } + + private const string SourceCode = """ +#file RetryTests.csproj + + + + Exe + true + $TargetFrameworks$ + + + + + + + + + + PreserveNewest + + + + +#file UnitTest1.cs +using System; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + private static int _count1; + private static int _count2; + private static int _count3; + private static int _count4; + private static int _count5; + + [TestMethod] + [Retry(3)] + public void TestMethod1() + { + _count1++; + } + + [TestMethod] + [Retry(3)] + public void TestMethod2() + { + _count2++; + // This will fail TestMethod2 once. Then the first retry attempt will work. + if (_count2 <= 1) Assert.Fail("Failing TestMethod2"); + } + + [TestMethod] + [Retry(3)] + public void TestMethod3() + { + _count3++; + // This will fail TestMethod3 twice. Then the second retry attempt will work. + if (_count3 <= 2) Assert.Fail("Failing TestMethod3"); + } + + [TestMethod] + [Retry(3)] + public void TestMethod4() + { + _count4++; + // This will fail TestMethod4 three times. Then the third retry attempt will work. + // In total, this test will run four times, and the last run will pass. + if (_count4 <= 3) Assert.Fail("Failing TestMethod4"); + } + + [TestMethod] + [Retry(3)] + public void TestMethod5() + { + _count5++; + // This will fail TestMethod5 four times. The end result is failure of this test. + Assert.Fail($"Failing TestMethod4. Attempts: {_count5}"); + } + + [ClassCleanup] + public static void ClassCleanup() + { + Console.WriteLine($"TestMethod1 executed {_count1} time."); + Console.WriteLine($"TestMethod2 executed {_count2} times."); + Console.WriteLine($"TestMethod3 executed {_count3} times."); + Console.WriteLine($"TestMethod4 executed {_count4} times."); + Console.WriteLine($"TestMethod5 executed {_count5} times."); + } +} + +#file my.runsettings + + + false + + +"""; + } +} diff --git a/test/IntegrationTests/MSTest.IntegrationTests/ClsTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/ClsTests.cs index 042cf09853..bd821151c9 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/ClsTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/ClsTests.cs @@ -11,14 +11,14 @@ public class ClsTests : CLITestBase // This test in itself is not so important. What matters is that the asset gets build. If we regress and start having // the [DataRow] attribute no longer CLS compliant, the build will raise a warning in VS (and the build will fail in CI). - public void TestsAreRun() + public async Task TestsAreRun() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( diff --git a/test/IntegrationTests/MSTest.IntegrationTests/DiscoverInternalsTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/DiscoverInternalsTests.cs index f26866c86e..2f3c14c430 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/DiscoverInternalsTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/DiscoverInternalsTests.cs @@ -9,14 +9,14 @@ public class DiscoverInternalsTests : CLITestBase { private const string TestAsset = "DiscoverInternalsProject"; - public void InternalTestClassesAreDiscoveredWhenTheDiscoverInternalsAttributeIsPresent() + public async Task InternalTestClassesAreDiscoveredWhenTheDiscoverInternalsAttributeIsPresent() { // Arrange string assemblyPath = Path.IsPathRooted(TestAsset) ? TestAsset : GetAssetFullPath(TestAsset); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath); - _ = RunTests(testCases); + _ = await RunTestsAsync(testCases); // Assert VerifyE2E.AtLeastTestsDiscovered( @@ -39,7 +39,7 @@ public void AnInternalTestClassDerivedFromAPublicAbstractGenericBaseClassForAnIn "EqualityIsCaseInsensitive"); } - public void AnInternalTypeCanBeUsedInADynamicDataTestMethod() + public async Task AnInternalTypeCanBeUsedInADynamicDataTestMethod() { string assemblyPath = Path.IsPathRooted(TestAsset) ? TestAsset : GetAssetFullPath(TestAsset); @@ -48,7 +48,7 @@ public void AnInternalTypeCanBeUsedInADynamicDataTestMethod() IEnumerable targetTestCases = testCases.Where(t => t.DisplayName == "DynamicDataTestMethod (DiscoverInternalsProject.SerializableInternalType)"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(targetTestCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(targetTestCases); // Assert VerifyE2E.TestsPassed( diff --git a/test/IntegrationTests/MSTest.IntegrationTests/FSharpTestProjectTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/FSharpTestProjectTests.cs index f26c80e6f4..59713dd4d3 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/FSharpTestProjectTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/FSharpTestProjectTests.cs @@ -9,14 +9,14 @@ public class FSharpTestProjectTests : CLITestBase { private const string TestAssetName = "FSharpTestProject"; - public void TestFSharpTestsWithSpaceAndDotInName() + public async Task TestFSharpTestsWithSpaceAndDotInName() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName, targetFramework: "net472"); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed(testResults, "Test method passing with a . in it"); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs index 1e8284032d..d074078f1d 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs @@ -12,11 +12,11 @@ public class OutputTests : CLITestBase { private const string TestAssetName = "OutputTestProject"; - public void OutputIsNotMixedWhenTestsRunInParallel() => ValidateOutputForClass("UnitTest1"); + public async Task OutputIsNotMixedWhenTestsRunInParallel() => await ValidateOutputForClass("UnitTest1"); - public void OutputIsNotMixedWhenAsyncTestsRunInParallel() => ValidateOutputForClass("UnitTest2"); + public async Task OutputIsNotMixedWhenAsyncTestsRunInParallel() => await ValidateOutputForClass("UnitTest2"); - private static void ValidateOutputForClass(string className) + private static async Task ValidateOutputForClass(string className) { // LogMessageListener uses an implementation of a string writer that captures output per async context. // This allows us to capture output from tasks even when they are running in parallel. @@ -29,7 +29,7 @@ private static void ValidateOutputForClass(string className) testCases.Should().HaveCount(3); testCases.Should().NotContainNulls(); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); testResults.Should().HaveCount(3); testResults.Should().NotContainNulls(); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs index 636ac3737e..42eb7e107a 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs @@ -18,53 +18,53 @@ public class DataExtensibilityTests : CLITestBase - Ignored tests are not expanded (DataRow, DataSource, etc) */ - public void CustomTestDataSourceTests() + public async Task CustomTestDataSourceTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomTestDataSourceTestMethod1"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsPassed(testResults, "CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)"); } - public void CustomEmptyTestDataSourceTests() + public async Task CustomEmptyTestDataSourceTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomEmptyTestDataSourceTestMethod"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsFailed(testResults, new string[] { null }); } - public void AssertExtensibilityTests() + public async Task AssertExtensibilityTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FxExtensibilityTestProject.AssertExTest"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsFailed(testResults, "BasicFailingAssertExtensionTest", "ChainedFailingAssertExtensionTest"); } - public void ExecuteCustomTestExtensibilityTests() + public async Task ExecuteCustomTestExtensibilityTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "(Name~CustomTestMethod1)|(Name~CustomTestClass1)"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsPassed( @@ -84,14 +84,14 @@ public void ExecuteCustomTestExtensibilityTests() "CustomTestClass1 - Execution number 3"); } - public void ExecuteCustomTestExtensibilityWithTestDataTests() + public async Task ExecuteCustomTestExtensibilityWithTestDataTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "Name~CustomTestMethod2"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -110,14 +110,14 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests() "CustomTestMethod2 (\"C\")"); } - public void WhenUsingCustomITestDataSourceWithExpansionDisabled_RespectSetting() + public async Task WhenUsingCustomITestDataSourceWithExpansionDisabled_RespectSetting() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomDisableExpansionTestDataSourceTestMethod1"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert Verify(testCases.Length == 1); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs index 4be71e0050..ce287aa5e5 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs @@ -9,14 +9,14 @@ public class DataRowTests : CLITestBase { private const string TestAssetName = "DataRowTestProject"; - public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRows_SimpleDataRows() + public async Task ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRows_SimpleDataRows() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -28,14 +28,14 @@ public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRow "DataRowTestMethod (\"DerivedString2\")"); } - public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_SimpleDataRows() + public async Task ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_SimpleDataRows() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -44,14 +44,14 @@ public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_Simp "DataRowTestMethod (\"DerivedString2\")"); } - public void DataRowsExecuteWithRequiredAndOptionalParameters() + public async Task DataRowsExecuteWithRequiredAndOptionalParameters() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -61,14 +61,14 @@ public void DataRowsExecuteWithRequiredAndOptionalParameters() "DataRowTestMethodWithSomeOptionalParameters (123,\"DerivedOptionalString2\",\"DerivedOptionalString3\")"); } - public void DataRowsExecuteWithParamsArrayParameter() + public async Task DataRowsExecuteWithParamsArrayParameter() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -79,14 +79,14 @@ public void DataRowsExecuteWithParamsArrayParameter() "DataRowTestMethodWithParamsParameters (2,\"DerivedParamsArg1\",\"DerivedParamsArg2\",\"DerivedParamsArg3\")"); } - public void DataRowsFailWhenInvalidArgumentsProvided() + public async Task DataRowsFailWhenInvalidArgumentsProvided() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular&TestCategory~DataRowOptionalInvalidArguments"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsFailed( @@ -96,14 +96,14 @@ public void DataRowsFailWhenInvalidArgumentsProvided() "DataRowTestMethodFailsWithInvalidArguments (2,\"DerivedRequiredArgument\",\"DerivedOptionalArgument\",\"DerivedExtraArgument\")"); } - public void DataRowsShouldSerializeDoublesProperly() + public async Task DataRowsShouldSerializeDoublesProperly() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular.DataRowTestDouble"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -112,14 +112,14 @@ public void DataRowsShouldSerializeDoublesProperly() "DataRowTestDouble (10.02,20.02)"); } - public void DataRowsShouldSerializeMixedTypesProperly() + public async Task DataRowsShouldSerializeMixedTypesProperly() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowTestMixed"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -127,14 +127,14 @@ public void DataRowsShouldSerializeMixedTypesProperly() "DataRowTestMixed (10,10,10,10,10,10,10,\"10\")"); } - public void DataRowsShouldSerializeEnumsProperly() + public async Task DataRowsShouldSerializeEnumsProperly() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowEnums"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -145,14 +145,14 @@ public void DataRowsShouldSerializeEnumsProperly() "DataRowEnums (Gamma)"); } - public void DataRowsShouldHandleNonSerializableValues() + public async Task DataRowsShouldHandleNonSerializableValues() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowNonSerializable"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsDiscovered( @@ -166,14 +166,14 @@ public void DataRowsShouldHandleNonSerializableValues() "DataRowNonSerializable (DataRowTestProject.DataRowTests_DerivedClass)"); } - public void ExecuteDataRowTests_Enums() + public async Task ExecuteDataRowTests_Enums() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Enums"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -241,14 +241,14 @@ public void ExecuteDataRowTests_Enums() VerifyE2E.FailedTestCount(testResults, 0); } - public void ExecuteDataRowTests_NonSerializablePaths() + public async Task ExecuteDataRowTests_NonSerializablePaths() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_NonSerializablePaths"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -259,14 +259,14 @@ public void ExecuteDataRowTests_NonSerializablePaths() VerifyE2E.FailedTestCount(testResults, 0); } - public void ExecuteDataRowTests_Regular() + public async Task ExecuteDataRowTests_Regular() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -320,28 +320,28 @@ public void ExecuteDataRowTests_Regular() "DataRowTestMethodFailsWithInvalidArguments (2,\"DerivedRequiredArgument\",\"DerivedOptionalArgument\",\"DerivedExtraArgument\")"); } - public void GetDisplayName_AfterOverriding_GetsTheNewDisplayName() + public async Task GetDisplayName_AfterOverriding_GetsTheNewDisplayName() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenGetDisplayName"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); VerifyE2E.TestsPassed( testResults, "Overridden DisplayName"); } - public void ParameterizedTestsWithTestMethodSettingDisplayName_DataIsPrefixWithDisplayName() + public async Task ParameterizedTestsWithTestMethodSettingDisplayName_DataIsPrefixWithDisplayName() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenTestMethodDisplayNameForParameterizedTest"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); VerifyE2E.TestsPassed( testResults, diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataSourceTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataSourceTests.cs index cedbf48a14..beba2bdf32 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataSourceTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataSourceTests.cs @@ -11,7 +11,7 @@ public class DataSourceTests : CLITestBase // TODO @haplois | @evangelink: This test fails under CI - will be fixed in a future PR (Marked as private to ignore the test) #pragma warning disable IDE0051 // Remove unused private members - private void ExecuteCsvTestDataSourceTests() + private async Task ExecuteCsvTestDataSourceTests() #pragma warning restore IDE0051 // Remove unused private members { // Arrange @@ -19,7 +19,7 @@ private void ExecuteCsvTestDataSourceTests() // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "CsvTestMethod"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsPassed( diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs index 06d74ed0c7..bd9f53ecaf 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs @@ -12,14 +12,14 @@ public class DynamicDataTests : CLITestBase { private const string TestAssetName = "DynamicDataTestProject"; - public void ExecuteDynamicDataTests() + public async Task ExecuteDynamicDataTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DynamicDataTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -82,14 +82,14 @@ public void ExecuteDynamicDataTests() VerifyE2E.FailedTestCount(testResults, 0); } - public void ExecuteDynamicDataTestsWithCategoryFilter() + public async Task ExecuteDynamicDataTestsWithCategoryFilter() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DynamicDataWithCategory"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsPassed( @@ -100,14 +100,14 @@ public void ExecuteDynamicDataTestsWithCategoryFilter() VerifyE2E.FailedTestCount(testResults, 0); } - public void ExecuteNonExpandableDynamicDataTests() + public async Task ExecuteNonExpandableDynamicDataTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DisableExpansionTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert Verify(testCases.Length == 6); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/TestId.DefaultStrategy.cs b/test/IntegrationTests/MSTest.IntegrationTests/TestId.DefaultStrategy.cs index 924ac62427..dc33ae9ad5 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/TestId.DefaultStrategy.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/TestId.DefaultStrategy.cs @@ -11,14 +11,14 @@ public partial class TestId : CLITestBase { private const string DefaultStrategyDll = "TestIdProject.DefaultStrategy"; - public void TestIdUniqueness_DataRowArray_DefaultStrategy() + public async Task TestIdUniqueness_DataRowArray_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -32,14 +32,14 @@ public void TestIdUniqueness_DataRowArray_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DataRowString_DefaultStrategy() + public async Task TestIdUniqueness_DataRowString_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowStringTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -54,14 +54,14 @@ public void TestIdUniqueness_DataRowString_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataArrays_DefaultStrategy() + public async Task TestIdUniqueness_DynamicDataArrays_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -75,14 +75,14 @@ public void TestIdUniqueness_DynamicDataArrays_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataTuple_DefaultStrategy() + public async Task TestIdUniqueness_DynamicDataTuple_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -95,14 +95,14 @@ public void TestIdUniqueness_DynamicDataTuple_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataGenericCollections_DefaultStrategy() + public async Task TestIdUniqueness_DynamicDataGenericCollections_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -117,14 +117,14 @@ public void TestIdUniqueness_DynamicDataGenericCollections_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceArrays_DefaultStrategy() + public async Task TestIdUniqueness_TestDataSourceArrays_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -138,14 +138,14 @@ public void TestIdUniqueness_TestDataSourceArrays_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceTuples_DefaultStrategy() + public async Task TestIdUniqueness_TestDataSourceTuples_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -158,14 +158,14 @@ public void TestIdUniqueness_TestDataSourceTuples_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceGenericCollections_DefaultStrategy() + public async Task TestIdUniqueness_TestDataSourceGenericCollections_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/TestId.DisplayNameStrategy.cs b/test/IntegrationTests/MSTest.IntegrationTests/TestId.DisplayNameStrategy.cs index 503fa65dfc..5f927acbf4 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/TestId.DisplayNameStrategy.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/TestId.DisplayNameStrategy.cs @@ -11,14 +11,14 @@ public partial class TestId : CLITestBase { private const string DisplayNameStrategyDll = "TestIdProject.DisplayNameStrategy"; - public void TestIdUniqueness_DataRowArray_DisplayNameStrategy() + public async Task TestIdUniqueness_DataRowArray_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -32,14 +32,14 @@ public void TestIdUniqueness_DataRowArray_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DataRowString_DisplayNameStrategy() + public async Task TestIdUniqueness_DataRowString_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowStringTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -55,14 +55,14 @@ public void TestIdUniqueness_DataRowString_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().HaveCount(3); } - public void TestIdUniqueness_DynamicDataArrays_DisplayNameStrategy() + public async Task TestIdUniqueness_DynamicDataArrays_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -76,14 +76,14 @@ public void TestIdUniqueness_DynamicDataArrays_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DynamicDataTuple_DisplayNameStrategy() + public async Task TestIdUniqueness_DynamicDataTuple_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -96,14 +96,14 @@ public void TestIdUniqueness_DynamicDataTuple_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataGenericCollections_DisplayNameStrategy() + public async Task TestIdUniqueness_DynamicDataGenericCollections_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -118,14 +118,14 @@ public void TestIdUniqueness_DynamicDataGenericCollections_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceArrays_DisplayNameStrategy() + public async Task TestIdUniqueness_TestDataSourceArrays_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -139,14 +139,14 @@ public void TestIdUniqueness_TestDataSourceArrays_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceTuples_DisplayNameStrategy() + public async Task TestIdUniqueness_TestDataSourceTuples_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -159,14 +159,14 @@ public void TestIdUniqueness_TestDataSourceTuples_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceGenericCollections_DisplayNameStrategy() + public async Task TestIdUniqueness_TestDataSourceGenericCollections_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/TestId.FullyQualifiedStrategy.cs b/test/IntegrationTests/MSTest.IntegrationTests/TestId.FullyQualifiedStrategy.cs index 4842ea364d..20512351bb 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/TestId.FullyQualifiedStrategy.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/TestId.FullyQualifiedStrategy.cs @@ -13,14 +13,14 @@ public partial class TestId : CLITestBase { private const string FullyQualifiedStrategyDll = "TestIdProject.FullyQualifiedStrategy"; - public void TestIdUniqueness_DataRowArray_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DataRowArray_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowArraysTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -34,14 +34,14 @@ public void TestIdUniqueness_DataRowArray_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DataRowString_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DataRowString_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowStringTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -56,14 +56,14 @@ public void TestIdUniqueness_DataRowString_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataArrays_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DynamicDataArrays_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataArraysTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -77,14 +77,14 @@ public void TestIdUniqueness_DynamicDataArrays_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataTuple_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DynamicDataTuple_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataTuplesTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -97,14 +97,14 @@ public void TestIdUniqueness_DynamicDataTuple_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataGenericCollections_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DynamicDataGenericCollections_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataGenericCollectionsTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -119,14 +119,14 @@ public void TestIdUniqueness_DynamicDataGenericCollections_FullyQualifiedStrateg testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceArrays_FullyQualifiedStrategy() + public async Task TestIdUniqueness_TestDataSourceArrays_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceArraysTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -140,14 +140,14 @@ public void TestIdUniqueness_TestDataSourceArrays_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceTuples_FullyQualifiedStrategy() + public async Task TestIdUniqueness_TestDataSourceTuples_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceTuplesTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -160,14 +160,14 @@ public void TestIdUniqueness_TestDataSourceTuples_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceGenericCollections_FullyQualifiedStrategy() + public async Task TestIdUniqueness_TestDataSourceGenericCollections_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceGenericCollectionsTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/TestId.LegacyStrategy.cs b/test/IntegrationTests/MSTest.IntegrationTests/TestId.LegacyStrategy.cs index 4f53cbfdd3..db0595e7bd 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/TestId.LegacyStrategy.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/TestId.LegacyStrategy.cs @@ -11,14 +11,14 @@ public partial class TestId : CLITestBase { private const string LegacyStrategyDll = "TestIdProject.LegacyStrategy"; - public void TestIdUniqueness_DataRowArray_LegacyStrategy() + public async Task TestIdUniqueness_DataRowArray_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -33,14 +33,14 @@ public void TestIdUniqueness_DataRowArray_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DataRowString_LegacyStrategy() + public async Task TestIdUniqueness_DataRowString_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowStringTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -56,14 +56,14 @@ public void TestIdUniqueness_DataRowString_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DynamicDataArrays_LegacyStrategy() + public async Task TestIdUniqueness_DynamicDataArrays_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -78,14 +78,14 @@ public void TestIdUniqueness_DynamicDataArrays_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DynamicDataTuple_LegacyStrategy() + public async Task TestIdUniqueness_DynamicDataTuple_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -99,14 +99,14 @@ public void TestIdUniqueness_DynamicDataTuple_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DynamicDataGenericCollections_LegacyStrategy() + public async Task TestIdUniqueness_DynamicDataGenericCollections_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -122,14 +122,14 @@ public void TestIdUniqueness_DynamicDataGenericCollections_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceArrays_LegacyStrategy() + public async Task TestIdUniqueness_TestDataSourceArrays_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -144,14 +144,14 @@ public void TestIdUniqueness_TestDataSourceArrays_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceTuples_LegacyStrategy() + public async Task TestIdUniqueness_TestDataSourceTuples_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -165,14 +165,14 @@ public void TestIdUniqueness_TestDataSourceTuples_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceGenericCollections_LegacyStrategy() + public async Task TestIdUniqueness_TestDataSourceGenericCollections_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs b/test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs index 31d9057b6f..94367a837a 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs @@ -33,12 +33,12 @@ internal static ImmutableArray DiscoverTests(string assemblyPath, stri return sink.DiscoveredTests; } - internal static ImmutableArray RunTests(IEnumerable testCases) + internal static async Task> RunTestsAsync(IEnumerable testCases) { var testExecutionManager = new TestExecutionManager(); var frameworkHandle = new InternalFrameworkHandle(); - testExecutionManager.ExecuteTests(testCases, null, frameworkHandle, false); + await testExecutionManager.ExecuteTestsAsync(testCases, null, frameworkHandle, false); return frameworkHandle.GetFlattenedTestResults(); } diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs index f00010e440..0e70dfcfab 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs @@ -61,15 +61,11 @@ public TestExecutionManagerTests() _testExecutionManager = new TestExecutionManager( new EnvironmentWrapper(), - action => Task.Factory.StartNew( - () => - { - _enqueuedParallelTestsCount++; - action(); - }, - CancellationToken.None, - TaskCreationOptions.LongRunning, - TaskScheduler.Default)); + task => + { + _enqueuedParallelTestsCount++; + return task(); + }); } protected override void Dispose(bool disposing) @@ -1137,7 +1133,11 @@ internal class TestableTestExecutionManager : TestExecutionManager { internal Action, IRunContext, IFrameworkHandle, bool> ExecuteTestsWrapper { get; set; } - internal override void ExecuteTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) => ExecuteTestsWrapper?.Invoke(tests, runContext, frameworkHandle, isDeploymentDone); + internal override Task ExecuteTestsAsync(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) + { + ExecuteTestsWrapper?.Invoke(tests, runContext, frameworkHandle, isDeploymentDone); + return Task.CompletedTask; + } internal override UnitTestDiscoverer GetUnitTestDiscoverer() => new TestableUnitTestDiscoverer(); } diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs index 400d28d913..8a8db74a2f 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodInfoTests.cs @@ -1258,7 +1258,7 @@ public void ResolveExpectedExceptionShouldThrowWhenAttributeIsDefinedTwice_Diffe new TestAssemblyInfo(typeof(DummyTestClassForExpectedException).Assembly)); TypeInspectionException ex = UTF.Assert.ThrowsException(() => new TestMethodInfo(testMethodInfo, classInfo, _testMethodOptions)); - UTF.Assert.AreEqual("The test method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestMethodInfoTests+DummyTestClassForExpectedException.DummyTestMethod1 has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed.", ex.Message); + UTF.Assert.AreEqual("The test method 'Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestMethodInfoTests+DummyTestClassForExpectedException.DummyTestMethod1' has multiple attributes derived from 'ExpectedExceptionBaseAttribute' defined on it. Only one such attribute is allowed.", ex.Message); } public void ResolveExpectedExceptionShouldThrowWhenAttributeIsDefinedTwice_SameConcreteType() @@ -1272,7 +1272,7 @@ public void ResolveExpectedExceptionShouldThrowWhenAttributeIsDefinedTwice_SameC new TestAssemblyInfo(typeof(DummyTestClassForExpectedException).Assembly)); TypeInspectionException ex = UTF.Assert.ThrowsException(() => new TestMethodInfo(testMethodInfo, classInfo, _testMethodOptions)); - UTF.Assert.AreEqual("The test method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestMethodInfoTests+DummyTestClassForExpectedException.DummyTestMethod1 has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed.", ex.Message); + UTF.Assert.AreEqual("The test method 'Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestMethodInfoTests+DummyTestClassForExpectedException.DummyTestMethod1' has multiple attributes derived from 'ExpectedExceptionBaseAttribute' defined on it. Only one such attribute is allowed.", ex.Message); } public void ResolveExpectedExceptionHelperShouldReturnExpectedExceptionAttributeIfPresent() diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TypeCacheTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TypeCacheTests.cs index 3e0bd1ccb3..2398bd2457 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TypeCacheTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TypeCacheTests.cs @@ -1337,8 +1337,8 @@ public void ResolveExpectedExceptionHelperShouldThrowIfMultipleExpectedException } catch (Exception ex) { - string message = "The test method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TypeCacheTests+DummyTestClassWithTestMethods.TestMethodWithMultipleExpectedException " - + "has multiple attributes derived from ExpectedExceptionBaseAttribute defined on it. Only one such attribute is allowed."; + string message = "The test method 'Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TypeCacheTests+DummyTestClassWithTestMethods.TestMethodWithMultipleExpectedException' " + + "has multiple attributes derived from 'ExpectedExceptionBaseAttribute' defined on it. Only one such attribute is allowed."; Verify(ex.Message == message); } } diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/UnitTestRunnerTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/UnitTestRunnerTests.cs index 52524eabf9..d9941fcd5a 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/UnitTestRunnerTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/UnitTestRunnerTests.cs @@ -80,23 +80,23 @@ public void ConstructorShouldPopulateSettings() #region RunSingleTest tests - public void RunSingleTestShouldThrowIfTestMethodIsNull() => - VerifyThrows(() => _unitTestRunner.RunSingleTest(null, null, null)); + public async Task RunSingleTestShouldThrowIfTestMethodIsNull() => + await VerifyThrowsAsync(async () => await _unitTestRunner.RunSingleTestAsync(null, null, null)); - public void RunSingleTestShouldThrowIfTestRunParametersIsNull() + public async Task RunSingleTestShouldThrowIfTestRunParametersIsNull() { var testMethod = new TestMethod("M", "C", "A", isAsync: false); - VerifyThrows(() => _unitTestRunner.RunSingleTest(testMethod, null, null)); + await VerifyThrowsAsync(async () => await _unitTestRunner.RunSingleTestAsync(testMethod, null, null)); } - public void RunSingleTestShouldReturnTestResultIndicateATestNotFoundIfTestMethodCannotBeFound() + public async Task RunSingleTestShouldReturnTestResultIndicateATestNotFoundIfTestMethodCannotBeFound() { var testMethod = new TestMethod("M", "C", "A", isAsync: false); _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -104,7 +104,7 @@ public void RunSingleTestShouldReturnTestResultIndicateATestNotFoundIfTestMethod Verify(results[0].ErrorMessage == "Test method M was not found."); } - public void RunSingleTestShouldReturnTestResultIndicatingNotRunnableTestIfTestMethodCannotBeRun() + public async Task RunSingleTestShouldReturnTestResultIndicatingNotRunnableTestIfTestMethodCannotBeRun() { Type type = typeof(TypeCacheTests.DummyTestClassWithTestMethods); MethodInfo methodInfo = type.GetMethod("TestMethodWithNullCustomPropertyName"); @@ -113,7 +113,7 @@ public void RunSingleTestShouldReturnTestResultIndicatingNotRunnableTestIfTestMe _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); string expectedMessage = string.Format( CultureInfo.InvariantCulture, @@ -127,7 +127,7 @@ public void RunSingleTestShouldReturnTestResultIndicatingNotRunnableTestIfTestMe Verify(expectedMessage == results[0].ErrorMessage); } - public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsPresentOnTestClassAndHasMessage() + public async Task ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsPresentOnTestClassAndHasMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreClassWithMessage); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -136,7 +136,7 @@ public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsP _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -144,7 +144,7 @@ public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsP Verify(results[0].ErrorMessage == "IgnoreTestClassMessage"); } - public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsPresentOnTestClassButHasNoMessage() + public async Task ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsPresentOnTestClassButHasNoMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreClass); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -153,7 +153,7 @@ public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsP _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -161,7 +161,7 @@ public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsP Verify(results[0].ErrorMessage == string.Empty); } - public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIsPresentOnTestMethodAndHasMessage() + public async Task ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIsPresentOnTestMethodAndHasMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreTestWithMessage); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -170,7 +170,7 @@ public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIs _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -178,7 +178,7 @@ public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIs Verify(results[0].ErrorMessage == "IgnoreTestMessage"); } - public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsPresentOnTestMethodButHasNoMessage() + public async Task ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsPresentOnTestMethodButHasNoMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreTest); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -187,7 +187,7 @@ public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsP _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -195,7 +195,7 @@ public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsP Verify(results[0].ErrorMessage == string.Empty); } - public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsPresentOnBothClassAndMethod() + public async Task ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsPresentOnBothClassAndMethod() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreClassAndIgnoreTestWithMessage); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -204,7 +204,7 @@ public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsP _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -212,7 +212,7 @@ public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsP Verify(results[0].ErrorMessage == "IgnoreTestClassMessage"); } - public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIsPresentOnBothClassAndMethodButClassHasNoMessage() + public async Task ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIsPresentOnBothClassAndMethodButClassHasNoMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreClassWithNoMessageAndIgnoreTestWithMessage); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -221,7 +221,7 @@ public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIs _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -229,7 +229,7 @@ public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIs Verify(results[0].ErrorMessage == "IgnoreTestMessage"); } - public void RunSingleTestShouldReturnTestResultIndicatingFailureIfThereIsAnyTypeInspectionExceptionWhenInspectingTestMethod() + public async Task RunSingleTestShouldReturnTestResultIndicatingFailureIfThereIsAnyTypeInspectionExceptionWhenInspectingTestMethod() { Type type = typeof(TypeCacheTests.DummyTestClassWithTestMethods); var testMethod = new TestMethod("ImaginaryTestMethod", type.FullName, "A", isAsync: false); @@ -237,7 +237,7 @@ public void RunSingleTestShouldReturnTestResultIndicatingFailureIfThereIsAnyType _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); string expectedMessage = string.Format( CultureInfo.InvariantCulture, @@ -251,7 +251,7 @@ public void RunSingleTestShouldReturnTestResultIndicatingFailureIfThereIsAnyType Verify(expectedMessage == results[0].ErrorMessage); } - public void RunSingleTestShouldReturnTestResultsForAPassingTestMethod() + public async Task RunSingleTestShouldReturnTestResultsForAPassingTestMethod() { Type type = typeof(TypeCacheTests.DummyTestClassWithTestMethods); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -260,7 +260,7 @@ public void RunSingleTestShouldReturnTestResultsForAPassingTestMethod() _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -268,7 +268,7 @@ public void RunSingleTestShouldReturnTestResultsForAPassingTestMethod() Verify(results[0].ErrorMessage is null); } - public void RunSingleTestShouldSetTestsAsInProgressInTestContext() + public async Task RunSingleTestShouldSetTestsAsInProgressInTestContext() { Type type = typeof(DummyTestClass); MethodInfo methodInfo = type.GetMethod("TestMethodToTestInProgress"); @@ -278,14 +278,14 @@ public void RunSingleTestShouldSetTestsAsInProgressInTestContext() .Returns(Assembly.GetExecutingAssembly()); // Asserting in the test method execution flow itself. - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); Verify(results[0].Outcome == UnitTestOutcome.Passed); } - public void RunSingleTestShouldCallAssemblyInitializeAndClassInitializeMethodsInOrder() + public async Task RunSingleTestShouldCallAssemblyInitializeAndClassInitializeMethodsInOrder() { var mockReflectHelper = new Mock(); _unitTestRunner = new UnitTestRunner(new MSTestSettings(), Array.Empty(), null, mockReflectHelper.Object); @@ -304,7 +304,7 @@ public void RunSingleTestShouldCallAssemblyInitializeAndClassInitializeMethodsIn DummyTestClassWithInitializeMethods.AssemblyInitializeMethodBody = () => validator <<= 2; DummyTestClassWithInitializeMethods.ClassInitializeMethodBody = () => validator >>= 2; - _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(validator == 1); }