diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs index a238267e72..ff9d9cf6c1 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs @@ -39,7 +39,7 @@ public void SendMessage(TestMessageLevel testMessageLevel, string message) /// private readonly IDictionary _sessionParameters; private readonly IEnvironment _environment; - private readonly Func _taskFactory; + private readonly Func, Task> _taskFactory; /// /// Specifies whether the test run is canceled or not. @@ -51,7 +51,7 @@ public TestExecutionManager() { } - internal TestExecutionManager(IEnvironment environment, Func? taskFactory = null) + internal TestExecutionManager(IEnvironment environment, Func, Task>? taskFactory = null) { _testMethodFilter = new TestMethodFilter(); _sessionParameters = new Dictionary(); @@ -59,7 +59,7 @@ internal TestExecutionManager(IEnvironment environment, Func? task _taskFactory = taskFactory ?? DefaultFactoryAsync; } - private static Task DefaultFactoryAsync(Action action) + private static Task DefaultFactoryAsync(Func taskGetter) { if (MSTestSettings.RunConfigurationSettings.ExecutionApartmentState == ApartmentState.STA && RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -69,7 +69,9 @@ private static Task DefaultFactoryAsync(Action action) { try { - action(); + // This is best we can do to execute in STA thread. + Task task = taskGetter(); + task.GetAwaiter().GetResult(); tcs.SetResult(0); } catch (Exception ex) @@ -84,7 +86,7 @@ private static Task DefaultFactoryAsync(Action action) } else { - return Task.Run(action); + return taskGetter(); } } @@ -121,7 +123,9 @@ public void RunTests(IEnumerable tests, IRunContext? runContext, IFram CacheSessionParameters(runContext, frameworkHandle); // Execute the tests - ExecuteTests(tests, runContext, frameworkHandle, isDeploymentDone); + // This is a public API, so we can't change it to be async. + // Consider not using this API internally, and introduce an async version, and mark this as obsolete. + ExecuteTestsAsync(tests, runContext, frameworkHandle, isDeploymentDone).GetAwaiter().GetResult(); if (!_hasAnyTestFailed) { @@ -159,7 +163,9 @@ public void RunTests(IEnumerable sources, IRunContext? runContext, IFram CacheSessionParameters(runContext, frameworkHandle); // Run tests. - ExecuteTests(tests, runContext, frameworkHandle, isDeploymentDone); + // This is a public API, so we can't change it to be async. + // Consider not using this API internally, and introduce an async version, and mark this as obsolete. + ExecuteTestsAsync(tests, runContext, frameworkHandle, isDeploymentDone).GetAwaiter().GetResult(); if (!_hasAnyTestFailed) { @@ -174,7 +180,7 @@ public void RunTests(IEnumerable sources, IRunContext? runContext, IFram /// The run context. /// Handle to record test start/end/results. /// Indicates if deployment is done. - internal virtual void ExecuteTests(IEnumerable tests, IRunContext? runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) + internal virtual async Task ExecuteTestsAsync(IEnumerable tests, IRunContext? runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) { var testsBySource = from test in tests group test by test.Source into testGroup @@ -183,7 +189,7 @@ group test by test.Source into testGroup foreach (var group in testsBySource) { _testRunCancellationToken?.ThrowIfCancellationRequested(); - ExecuteTestsInSource(group.Tests, runContext, frameworkHandle, group.Source, isDeploymentDone); + await ExecuteTestsInSourceAsync(group.Tests, runContext, frameworkHandle, group.Source, isDeploymentDone); } } @@ -257,7 +263,7 @@ private static bool MatchTestFilter(ITestCaseFilterExpression? filterExpression, /// Handle to record test start/end/results. /// The test container for the tests. /// Indicates if deployment is done. - private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone) + private async Task ExecuteTestsInSourceAsync(IEnumerable tests, IRunContext? runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone) { DebugEx.Assert(!StringEx.IsNullOrEmpty(source), "Source cannot be empty"); @@ -363,7 +369,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC { _testRunCancellationToken?.ThrowIfCancellationRequested(); - tasks.Add(_taskFactory(() => + tasks.Add(_taskFactory(async () => { try { @@ -373,7 +379,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC if (queue.TryDequeue(out IEnumerable? testSet)) { - ExecuteTestsWithTestRunner(testSet, frameworkHandle, source, sourceLevelParameters, testRunner); + await ExecuteTestsWithTestRunnerAsync(testSet, frameworkHandle, source, sourceLevelParameters, testRunner); } } } @@ -385,7 +391,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC try { - Task.WaitAll(tasks.ToArray()); + await Task.WhenAll(tasks); } catch (Exception ex) { @@ -399,12 +405,12 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC // Queue the non parallel set if (nonParallelizableTestSet != null) { - ExecuteTestsWithTestRunner(nonParallelizableTestSet, frameworkHandle, source, sourceLevelParameters, testRunner); + await ExecuteTestsWithTestRunnerAsync(nonParallelizableTestSet, frameworkHandle, source, sourceLevelParameters, testRunner); } } else { - ExecuteTestsWithTestRunner(testsToRun, frameworkHandle, source, sourceLevelParameters, testRunner); + await ExecuteTestsWithTestRunnerAsync(testsToRun, frameworkHandle, source, sourceLevelParameters, testRunner); } if (PlatformServiceProvider.Instance.IsGracefulStopRequested) @@ -415,7 +421,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed tests belonging to source {0}", source); } - private void ExecuteTestsWithTestRunner( + private Task ExecuteTestsWithTestRunnerAsync( IEnumerable tests, ITestExecutionRecorder testExecutionRecorder, string source, @@ -462,7 +468,10 @@ private void ExecuteTestsWithTestRunner( // testRunner could be in a different AppDomain. We cannot pass the testExecutionRecorder directly. // Instead, we pass a proxy (remoting object) that is marshallable by ref. +#pragma warning disable VSTHRD103 // Call async methods when in an async method - We cannot do right now because we are crossing app domains. + // TODO: Use await testRunner.RunSingleTestAsync. UnitTestResult[] unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testContextProperties, remotingMessageLogger); +#pragma warning restore VSTHRD103 // Call async methods when in an async method PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed test {0}", unitTestElement.TestMethod.Name); @@ -496,6 +505,8 @@ private void ExecuteTestsWithTestRunner( SendTestResults(currentTest, [result], DateTimeOffset.Now, DateTimeOffset.Now, testExecutionRecorder); } } + + return Task.CompletedTask; } /// diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs index 3b61f1aae0..f63c20cae3 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodInfo.cs @@ -46,6 +46,7 @@ internal TestMethodInfo( Parent = parent; TestMethodOptions = testMethodOptions; ExpectedException = ResolveExpectedException(); + RetryAttribute = GetRetryAttribute(); } /// @@ -92,6 +93,8 @@ internal TestMethodInfo( internal ExpectedExceptionBaseAttribute? ExpectedException { get; set; /*set for testing only*/ } + internal RetryAttribute? RetryAttribute { get; } + public Attribute[]? GetAllAttributes(bool inherit) => ReflectHelper.Instance.GetDerivedAttributes(TestMethod, inherit).ToArray(); public TAttributeType[] GetAttributes(bool inherit) @@ -260,6 +263,16 @@ public virtual TestResult Invoke(object?[]? arguments) return expectedExceptions.FirstOrDefault(); } + /// + /// Gets the number of retries this test method should make in case of failure. + /// + /// + /// The number of retries, which is always greater than or equal to 1. + /// If RetryAttribute is not present, returns 1. + /// + private RetryAttribute? GetRetryAttribute() + => ReflectHelper.Instance.GetFirstDerivedAttributeOrDefault(TestMethod, inherit: true); + /// /// Execute test without timeout. /// diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs index bbc8f4f0a5..9d65b94165 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs @@ -173,6 +173,7 @@ internal List RunTestMethod() { if (_test.TestDataSourceIgnoreMessage is not null) { + _testContext.SetOutcome(UTF.UnitTestOutcome.Ignored); return [new() { Outcome = UTF.UnitTestOutcome.Ignored, IgnoreReason = _test.TestDataSourceIgnoreMessage }]; } diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index 563085f288..9d4794eb6a 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -6,6 +6,7 @@ using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -122,13 +123,19 @@ internal FixtureTestResult GetFixtureTestResult(TestMethod testMethod, string fi static UnitTestOutcome GetOutcome(Exception? exception) => exception == null ? UnitTestOutcome.Passed : UnitTestOutcome.Failed; } + // Task cannot cross app domains. + // For now, TestExecutionManager will call this sync method which is hacky. + // If we removed AppDomains in v4, we should use the async method and remove this one. + internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary testContextProperties, IMessageLogger messageLogger) + => RunSingleTestAsync(testMethod, testContextProperties, messageLogger).GetAwaiter().GetResult(); + /// /// Runs a single test. /// /// The test Method. /// The test context properties. /// The . - internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary testContextProperties, IMessageLogger messageLogger) + internal async Task RunSingleTestAsync(TestMethod testMethod, IDictionary testContextProperties, IMessageLogger messageLogger) { Guard.NotNull(testMethod); @@ -179,8 +186,23 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary firstRunResult = testMethodRunner.Execute(classInitializeResult.StandardOut!, classInitializeResult.StandardError!, classInitializeResult.DebugTrace!, classInitializeResult.TestContextMessages!); + result = firstRunResult.ToUnitTestResults(); + if (retryAttribute is not null && !RetryAttribute.IsAcceptableResultForRetry(firstRunResult)) + { + RetryResult retryResult = await retryAttribute.ExecuteAsync( + new RetryContext( + () => Task.FromResult( + testMethodRunner.Execute( + classInitializeResult.StandardOut!, + classInitializeResult.StandardError!, + classInitializeResult.DebugTrace!, + classInitializeResult.TestContextMessages!).ToArray()))); + + result = retryResult.TryGetLast()?.ToUnitTestResults() ?? throw ApplicationStateGuard.Unreachable(); + } } } } diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/DelayBackoffType.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/DelayBackoffType.cs new file mode 100644 index 0000000000..416d86b363 --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/DelayBackoffType.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Specifies a backoff type for the delay between retries. +/// +public enum DelayBackoffType +{ + /// + /// Specifies a constant backoff type. Meaning the delay between retries is constant. + /// + Constant, + + /// + /// Specifies an exponential backoff type. + /// The delay is calculated as the base delay * 2^(n-1) where n is the retry attempt. + /// For example, if the base delay is 1000ms, the delays will be 1000ms, 2000ms, 4000ms, 8000ms, etc. + /// + Exponential, +} diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/RetryAttribute.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryAttribute.cs new file mode 100644 index 0000000000..f892961c45 --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryAttribute.cs @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// This attribute is used to set a retry count on a test method in case of failure. +/// +[AttributeUsage(AttributeTargets.Method, Inherited = false, AllowMultiple = false)] +public class RetryAttribute : Attribute +{ + /// + /// Initializes a new instance of the class with the given number of max retries. + /// + /// The maximum number of retry attempts. This must be greater than or equal to 1. + public RetryAttribute(int maxRetryAttempts) + { + if (maxRetryAttempts < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxRetryAttempts)); + } + + MaxRetryAttempts = maxRetryAttempts; + } + + /// + /// Gets the number of retries that the test should make in case of failures. + /// Note that before RetryAttribute is considered, the test was already executed once. + /// This property determines the max number of retries after the first normal run. + /// + public int MaxRetryAttempts { get; } + + /// + /// Gets or sets the delay, in milliseconds, between retries. + /// This delay is also applied after the first run and before the first retry attempt. + /// + public int MillisecondsDelayBetweenRetries { get; set; } + + public DelayBackoffType BackoffType + { + get => field; + set + { + if (value is < DelayBackoffType.Constant or > DelayBackoffType.Exponential) + { + throw new ArgumentOutOfRangeException(nameof(value)); + } + + field = value; + } + } + + /// + /// Retries the test method times in case of failure. + /// Note that a first run of the method was already executed and failed before this method is called. + /// + /// An object to encapsulate the state needed for retry execution. + /// + /// Returns a object that contains the results of all attempts. Only + /// the last added element is used to determine the test outcome. + /// The other results are currently not used, but may be used in the future for tooling to show the + /// state of the failed attempts. + /// + protected internal virtual async Task ExecuteAsync(RetryContext retryContext) + { + var result = new RetryResult(); + int currentDelay = MillisecondsDelayBetweenRetries; + for (int i = 0; i < MaxRetryAttempts; i++) + { + // The caller already executed the test once. So we need to do the delay here. + await Task.Delay(currentDelay); + if (BackoffType == DelayBackoffType.Exponential) + { + currentDelay *= 2; + } + + TestResult[] testResults = await retryContext.ExecuteTaskGetter(); + result.AddResult(testResults); + if (IsAcceptableResultForRetry(testResults)) + { + break; + } + } + + return result; + } + + internal static bool IsAcceptableResultForRetry(TestResult[] results) + { + foreach (TestResult result in results) + { + UnitTestOutcome outcome = result.Outcome; + if (outcome is UnitTestOutcome.Failed or UnitTestOutcome.Timeout) + { + return false; + } + } + + return true; + } + + internal static bool IsAcceptableResultForRetry(List results) + { + foreach (TestResult result in results) + { + UnitTestOutcome outcome = result.Outcome; + if (outcome is UnitTestOutcome.Failed or UnitTestOutcome.Timeout) + { + return false; + } + } + + return true; + } +} diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/RetryContext.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryContext.cs new file mode 100644 index 0000000000..46a05fd00a --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryContext.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +public sealed class RetryContext +{ + internal RetryContext(Func> executeTaskGetter) + => ExecuteTaskGetter = executeTaskGetter; + + public Func> ExecuteTaskGetter { get; } +} diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/RetryResult.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryResult.cs new file mode 100644 index 0000000000..7d1337450f --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/RetryResult.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +public sealed class RetryResult +{ + private readonly List _testResults = new(); + + public void AddResult(TestResult[] testResults) + => _testResults.Add(testResults); + + internal TestResult[]? TryGetLast() + => _testResults.Count > 0 ? _testResults[_testResults.Count - 1] : null; +} diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index 01ad108d22..2d255084ba 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -190,6 +190,9 @@ Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AssertThrowsExactlyInterpola Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AssertThrowsExactlyInterpolatedStringHandler.AssertThrowsExactlyInterpolatedStringHandler(int literalLength, int formattedCount, System.Action! action, out bool shouldAppend) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IgnoreMessage.get -> string? Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IgnoreMessage.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType +Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType.Constant = 0 -> Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType +Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType.Exponential = 1 -> Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType) -> void @@ -202,6 +205,18 @@ Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.AutoDetect = Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.IgnoreMessage.get -> string? Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.IgnoreMessage.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.BackoffType.get -> Microsoft.VisualStudio.TestTools.UnitTesting.DelayBackoffType +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.BackoffType.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.MaxRetryAttempts.get -> int +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.MillisecondsDelayBetweenRetries.get -> int +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.MillisecondsDelayBetweenRetries.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.RetryAttribute(int maxRetryAttempts) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryContext +Microsoft.VisualStudio.TestTools.UnitTesting.RetryContext.ExecuteTaskGetter.get -> System.Func!>! +Microsoft.VisualStudio.TestTools.UnitTesting.RetryResult +Microsoft.VisualStudio.TestTools.UnitTesting.RetryResult.AddResult(Microsoft.VisualStudio.TestTools.UnitTesting.TestResult![]! testResults) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.RetryResult.RetryResult() -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.IgnoreMessage.get -> string? Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.IgnoreMessage.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.IgnoreMessage.get -> string? @@ -247,3 +262,4 @@ static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactly(System.Action! action, System.Func! messageBuilder) -> TException! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactlyAsync(System.Func! action, string! message = "", params object![]! messageArgs) -> System.Threading.Tasks.Task! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactlyAsync(System.Func! action, System.Func! messageBuilder) -> System.Threading.Tasks.Task! +virtual Microsoft.VisualStudio.TestTools.UnitTesting.RetryAttribute.ExecuteAsync(Microsoft.VisualStudio.TestTools.UnitTesting.RetryContext! retryContext) -> System.Threading.Tasks.Task! diff --git a/test/IntegrationTests/MSTest.IntegrationTests/ClsTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/ClsTests.cs index 042cf09853..bd821151c9 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/ClsTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/ClsTests.cs @@ -11,14 +11,14 @@ public class ClsTests : CLITestBase // This test in itself is not so important. What matters is that the asset gets build. If we regress and start having // the [DataRow] attribute no longer CLS compliant, the build will raise a warning in VS (and the build will fail in CI). - public void TestsAreRun() + public async Task TestsAreRun() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( diff --git a/test/IntegrationTests/MSTest.IntegrationTests/DiscoverInternalsTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/DiscoverInternalsTests.cs index f26866c86e..2f3c14c430 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/DiscoverInternalsTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/DiscoverInternalsTests.cs @@ -9,14 +9,14 @@ public class DiscoverInternalsTests : CLITestBase { private const string TestAsset = "DiscoverInternalsProject"; - public void InternalTestClassesAreDiscoveredWhenTheDiscoverInternalsAttributeIsPresent() + public async Task InternalTestClassesAreDiscoveredWhenTheDiscoverInternalsAttributeIsPresent() { // Arrange string assemblyPath = Path.IsPathRooted(TestAsset) ? TestAsset : GetAssetFullPath(TestAsset); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath); - _ = RunTests(testCases); + _ = await RunTestsAsync(testCases); // Assert VerifyE2E.AtLeastTestsDiscovered( @@ -39,7 +39,7 @@ public void AnInternalTestClassDerivedFromAPublicAbstractGenericBaseClassForAnIn "EqualityIsCaseInsensitive"); } - public void AnInternalTypeCanBeUsedInADynamicDataTestMethod() + public async Task AnInternalTypeCanBeUsedInADynamicDataTestMethod() { string assemblyPath = Path.IsPathRooted(TestAsset) ? TestAsset : GetAssetFullPath(TestAsset); @@ -48,7 +48,7 @@ public void AnInternalTypeCanBeUsedInADynamicDataTestMethod() IEnumerable targetTestCases = testCases.Where(t => t.DisplayName == "DynamicDataTestMethod (DiscoverInternalsProject.SerializableInternalType)"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(targetTestCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(targetTestCases); // Assert VerifyE2E.TestsPassed( diff --git a/test/IntegrationTests/MSTest.IntegrationTests/FSharpTestProjectTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/FSharpTestProjectTests.cs index f26c80e6f4..59713dd4d3 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/FSharpTestProjectTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/FSharpTestProjectTests.cs @@ -9,14 +9,14 @@ public class FSharpTestProjectTests : CLITestBase { private const string TestAssetName = "FSharpTestProject"; - public void TestFSharpTestsWithSpaceAndDotInName() + public async Task TestFSharpTestsWithSpaceAndDotInName() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName, targetFramework: "net472"); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed(testResults, "Test method passing with a . in it"); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs index 1e8284032d..d074078f1d 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs @@ -12,11 +12,11 @@ public class OutputTests : CLITestBase { private const string TestAssetName = "OutputTestProject"; - public void OutputIsNotMixedWhenTestsRunInParallel() => ValidateOutputForClass("UnitTest1"); + public async Task OutputIsNotMixedWhenTestsRunInParallel() => await ValidateOutputForClass("UnitTest1"); - public void OutputIsNotMixedWhenAsyncTestsRunInParallel() => ValidateOutputForClass("UnitTest2"); + public async Task OutputIsNotMixedWhenAsyncTestsRunInParallel() => await ValidateOutputForClass("UnitTest2"); - private static void ValidateOutputForClass(string className) + private static async Task ValidateOutputForClass(string className) { // LogMessageListener uses an implementation of a string writer that captures output per async context. // This allows us to capture output from tasks even when they are running in parallel. @@ -29,7 +29,7 @@ private static void ValidateOutputForClass(string className) testCases.Should().HaveCount(3); testCases.Should().NotContainNulls(); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); testResults.Should().HaveCount(3); testResults.Should().NotContainNulls(); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs index 636ac3737e..42eb7e107a 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs @@ -18,53 +18,53 @@ public class DataExtensibilityTests : CLITestBase - Ignored tests are not expanded (DataRow, DataSource, etc) */ - public void CustomTestDataSourceTests() + public async Task CustomTestDataSourceTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomTestDataSourceTestMethod1"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsPassed(testResults, "CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)"); } - public void CustomEmptyTestDataSourceTests() + public async Task CustomEmptyTestDataSourceTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomEmptyTestDataSourceTestMethod"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsFailed(testResults, new string[] { null }); } - public void AssertExtensibilityTests() + public async Task AssertExtensibilityTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FxExtensibilityTestProject.AssertExTest"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsFailed(testResults, "BasicFailingAssertExtensionTest", "ChainedFailingAssertExtensionTest"); } - public void ExecuteCustomTestExtensibilityTests() + public async Task ExecuteCustomTestExtensibilityTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "(Name~CustomTestMethod1)|(Name~CustomTestClass1)"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsPassed( @@ -84,14 +84,14 @@ public void ExecuteCustomTestExtensibilityTests() "CustomTestClass1 - Execution number 3"); } - public void ExecuteCustomTestExtensibilityWithTestDataTests() + public async Task ExecuteCustomTestExtensibilityWithTestDataTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "Name~CustomTestMethod2"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -110,14 +110,14 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests() "CustomTestMethod2 (\"C\")"); } - public void WhenUsingCustomITestDataSourceWithExpansionDisabled_RespectSetting() + public async Task WhenUsingCustomITestDataSourceWithExpansionDisabled_RespectSetting() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomDisableExpansionTestDataSourceTestMethod1"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert Verify(testCases.Length == 1); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs index 4be71e0050..ce287aa5e5 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs @@ -9,14 +9,14 @@ public class DataRowTests : CLITestBase { private const string TestAssetName = "DataRowTestProject"; - public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRows_SimpleDataRows() + public async Task ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRows_SimpleDataRows() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -28,14 +28,14 @@ public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRow "DataRowTestMethod (\"DerivedString2\")"); } - public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_SimpleDataRows() + public async Task ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_SimpleDataRows() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -44,14 +44,14 @@ public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_Simp "DataRowTestMethod (\"DerivedString2\")"); } - public void DataRowsExecuteWithRequiredAndOptionalParameters() + public async Task DataRowsExecuteWithRequiredAndOptionalParameters() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -61,14 +61,14 @@ public void DataRowsExecuteWithRequiredAndOptionalParameters() "DataRowTestMethodWithSomeOptionalParameters (123,\"DerivedOptionalString2\",\"DerivedOptionalString3\")"); } - public void DataRowsExecuteWithParamsArrayParameter() + public async Task DataRowsExecuteWithParamsArrayParameter() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -79,14 +79,14 @@ public void DataRowsExecuteWithParamsArrayParameter() "DataRowTestMethodWithParamsParameters (2,\"DerivedParamsArg1\",\"DerivedParamsArg2\",\"DerivedParamsArg3\")"); } - public void DataRowsFailWhenInvalidArgumentsProvided() + public async Task DataRowsFailWhenInvalidArgumentsProvided() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular&TestCategory~DataRowOptionalInvalidArguments"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsFailed( @@ -96,14 +96,14 @@ public void DataRowsFailWhenInvalidArgumentsProvided() "DataRowTestMethodFailsWithInvalidArguments (2,\"DerivedRequiredArgument\",\"DerivedOptionalArgument\",\"DerivedExtraArgument\")"); } - public void DataRowsShouldSerializeDoublesProperly() + public async Task DataRowsShouldSerializeDoublesProperly() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular.DataRowTestDouble"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -112,14 +112,14 @@ public void DataRowsShouldSerializeDoublesProperly() "DataRowTestDouble (10.02,20.02)"); } - public void DataRowsShouldSerializeMixedTypesProperly() + public async Task DataRowsShouldSerializeMixedTypesProperly() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowTestMixed"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -127,14 +127,14 @@ public void DataRowsShouldSerializeMixedTypesProperly() "DataRowTestMixed (10,10,10,10,10,10,10,\"10\")"); } - public void DataRowsShouldSerializeEnumsProperly() + public async Task DataRowsShouldSerializeEnumsProperly() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowEnums"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -145,14 +145,14 @@ public void DataRowsShouldSerializeEnumsProperly() "DataRowEnums (Gamma)"); } - public void DataRowsShouldHandleNonSerializableValues() + public async Task DataRowsShouldHandleNonSerializableValues() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowNonSerializable"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsDiscovered( @@ -166,14 +166,14 @@ public void DataRowsShouldHandleNonSerializableValues() "DataRowNonSerializable (DataRowTestProject.DataRowTests_DerivedClass)"); } - public void ExecuteDataRowTests_Enums() + public async Task ExecuteDataRowTests_Enums() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Enums"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -241,14 +241,14 @@ public void ExecuteDataRowTests_Enums() VerifyE2E.FailedTestCount(testResults, 0); } - public void ExecuteDataRowTests_NonSerializablePaths() + public async Task ExecuteDataRowTests_NonSerializablePaths() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_NonSerializablePaths"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -259,14 +259,14 @@ public void ExecuteDataRowTests_NonSerializablePaths() VerifyE2E.FailedTestCount(testResults, 0); } - public void ExecuteDataRowTests_Regular() + public async Task ExecuteDataRowTests_Regular() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -320,28 +320,28 @@ public void ExecuteDataRowTests_Regular() "DataRowTestMethodFailsWithInvalidArguments (2,\"DerivedRequiredArgument\",\"DerivedOptionalArgument\",\"DerivedExtraArgument\")"); } - public void GetDisplayName_AfterOverriding_GetsTheNewDisplayName() + public async Task GetDisplayName_AfterOverriding_GetsTheNewDisplayName() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenGetDisplayName"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); VerifyE2E.TestsPassed( testResults, "Overridden DisplayName"); } - public void ParameterizedTestsWithTestMethodSettingDisplayName_DataIsPrefixWithDisplayName() + public async Task ParameterizedTestsWithTestMethodSettingDisplayName_DataIsPrefixWithDisplayName() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenTestMethodDisplayNameForParameterizedTest"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); VerifyE2E.TestsPassed( testResults, diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataSourceTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataSourceTests.cs index cedbf48a14..beba2bdf32 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataSourceTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataSourceTests.cs @@ -11,7 +11,7 @@ public class DataSourceTests : CLITestBase // TODO @haplois | @evangelink: This test fails under CI - will be fixed in a future PR (Marked as private to ignore the test) #pragma warning disable IDE0051 // Remove unused private members - private void ExecuteCsvTestDataSourceTests() + private async Task ExecuteCsvTestDataSourceTests() #pragma warning restore IDE0051 // Remove unused private members { // Arrange @@ -19,7 +19,7 @@ private void ExecuteCsvTestDataSourceTests() // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "CsvTestMethod"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsPassed( diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs index 06d74ed0c7..bd9f53ecaf 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs @@ -12,14 +12,14 @@ public class DynamicDataTests : CLITestBase { private const string TestAssetName = "DynamicDataTestProject"; - public void ExecuteDynamicDataTests() + public async Task ExecuteDynamicDataTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DynamicDataTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.TestsPassed( @@ -82,14 +82,14 @@ public void ExecuteDynamicDataTests() VerifyE2E.FailedTestCount(testResults, 0); } - public void ExecuteDynamicDataTestsWithCategoryFilter() + public async Task ExecuteDynamicDataTestsWithCategoryFilter() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DynamicDataWithCategory"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.ContainsTestsPassed( @@ -100,14 +100,14 @@ public void ExecuteDynamicDataTestsWithCategoryFilter() VerifyE2E.FailedTestCount(testResults, 0); } - public void ExecuteNonExpandableDynamicDataTests() + public async Task ExecuteNonExpandableDynamicDataTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DisableExpansionTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert Verify(testCases.Length == 6); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/TestId.DefaultStrategy.cs b/test/IntegrationTests/MSTest.IntegrationTests/TestId.DefaultStrategy.cs index 924ac62427..dc33ae9ad5 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/TestId.DefaultStrategy.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/TestId.DefaultStrategy.cs @@ -11,14 +11,14 @@ public partial class TestId : CLITestBase { private const string DefaultStrategyDll = "TestIdProject.DefaultStrategy"; - public void TestIdUniqueness_DataRowArray_DefaultStrategy() + public async Task TestIdUniqueness_DataRowArray_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -32,14 +32,14 @@ public void TestIdUniqueness_DataRowArray_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DataRowString_DefaultStrategy() + public async Task TestIdUniqueness_DataRowString_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowStringTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -54,14 +54,14 @@ public void TestIdUniqueness_DataRowString_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataArrays_DefaultStrategy() + public async Task TestIdUniqueness_DynamicDataArrays_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -75,14 +75,14 @@ public void TestIdUniqueness_DynamicDataArrays_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataTuple_DefaultStrategy() + public async Task TestIdUniqueness_DynamicDataTuple_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -95,14 +95,14 @@ public void TestIdUniqueness_DynamicDataTuple_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataGenericCollections_DefaultStrategy() + public async Task TestIdUniqueness_DynamicDataGenericCollections_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -117,14 +117,14 @@ public void TestIdUniqueness_DynamicDataGenericCollections_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceArrays_DefaultStrategy() + public async Task TestIdUniqueness_TestDataSourceArrays_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -138,14 +138,14 @@ public void TestIdUniqueness_TestDataSourceArrays_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceTuples_DefaultStrategy() + public async Task TestIdUniqueness_TestDataSourceTuples_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -158,14 +158,14 @@ public void TestIdUniqueness_TestDataSourceTuples_DefaultStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceGenericCollections_DefaultStrategy() + public async Task TestIdUniqueness_TestDataSourceGenericCollections_DefaultStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DefaultStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/TestId.DisplayNameStrategy.cs b/test/IntegrationTests/MSTest.IntegrationTests/TestId.DisplayNameStrategy.cs index 503fa65dfc..5f927acbf4 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/TestId.DisplayNameStrategy.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/TestId.DisplayNameStrategy.cs @@ -11,14 +11,14 @@ public partial class TestId : CLITestBase { private const string DisplayNameStrategyDll = "TestIdProject.DisplayNameStrategy"; - public void TestIdUniqueness_DataRowArray_DisplayNameStrategy() + public async Task TestIdUniqueness_DataRowArray_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -32,14 +32,14 @@ public void TestIdUniqueness_DataRowArray_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DataRowString_DisplayNameStrategy() + public async Task TestIdUniqueness_DataRowString_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowStringTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -55,14 +55,14 @@ public void TestIdUniqueness_DataRowString_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().HaveCount(3); } - public void TestIdUniqueness_DynamicDataArrays_DisplayNameStrategy() + public async Task TestIdUniqueness_DynamicDataArrays_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -76,14 +76,14 @@ public void TestIdUniqueness_DynamicDataArrays_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DynamicDataTuple_DisplayNameStrategy() + public async Task TestIdUniqueness_DynamicDataTuple_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -96,14 +96,14 @@ public void TestIdUniqueness_DynamicDataTuple_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataGenericCollections_DisplayNameStrategy() + public async Task TestIdUniqueness_DynamicDataGenericCollections_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -118,14 +118,14 @@ public void TestIdUniqueness_DynamicDataGenericCollections_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceArrays_DisplayNameStrategy() + public async Task TestIdUniqueness_TestDataSourceArrays_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -139,14 +139,14 @@ public void TestIdUniqueness_TestDataSourceArrays_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceTuples_DisplayNameStrategy() + public async Task TestIdUniqueness_TestDataSourceTuples_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -159,14 +159,14 @@ public void TestIdUniqueness_TestDataSourceTuples_DisplayNameStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceGenericCollections_DisplayNameStrategy() + public async Task TestIdUniqueness_TestDataSourceGenericCollections_DisplayNameStrategy() { // Arrange string assemblyPath = GetAssetFullPath(DisplayNameStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/TestId.FullyQualifiedStrategy.cs b/test/IntegrationTests/MSTest.IntegrationTests/TestId.FullyQualifiedStrategy.cs index 4842ea364d..20512351bb 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/TestId.FullyQualifiedStrategy.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/TestId.FullyQualifiedStrategy.cs @@ -13,14 +13,14 @@ public partial class TestId : CLITestBase { private const string FullyQualifiedStrategyDll = "TestIdProject.FullyQualifiedStrategy"; - public void TestIdUniqueness_DataRowArray_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DataRowArray_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowArraysTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -34,14 +34,14 @@ public void TestIdUniqueness_DataRowArray_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DataRowString_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DataRowString_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowStringTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -56,14 +56,14 @@ public void TestIdUniqueness_DataRowString_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataArrays_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DynamicDataArrays_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataArraysTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -77,14 +77,14 @@ public void TestIdUniqueness_DynamicDataArrays_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataTuple_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DynamicDataTuple_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataTuplesTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -97,14 +97,14 @@ public void TestIdUniqueness_DynamicDataTuple_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_DynamicDataGenericCollections_FullyQualifiedStrategy() + public async Task TestIdUniqueness_DynamicDataGenericCollections_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataGenericCollectionsTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -119,14 +119,14 @@ public void TestIdUniqueness_DynamicDataGenericCollections_FullyQualifiedStrateg testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceArrays_FullyQualifiedStrategy() + public async Task TestIdUniqueness_TestDataSourceArrays_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceArraysTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -140,14 +140,14 @@ public void TestIdUniqueness_TestDataSourceArrays_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceTuples_FullyQualifiedStrategy() + public async Task TestIdUniqueness_TestDataSourceTuples_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceTuplesTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -160,14 +160,14 @@ public void TestIdUniqueness_TestDataSourceTuples_FullyQualifiedStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Should().OnlyHaveUniqueItems(); } - public void TestIdUniqueness_TestDataSourceGenericCollections_FullyQualifiedStrategy() + public async Task TestIdUniqueness_TestDataSourceGenericCollections_FullyQualifiedStrategy() { // Arrange string assemblyPath = GetAssetFullPath(FullyQualifiedStrategyDll); // Act ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceGenericCollectionsTests"); - ImmutableArray testResults = RunTests(testCases); + ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/TestId.LegacyStrategy.cs b/test/IntegrationTests/MSTest.IntegrationTests/TestId.LegacyStrategy.cs index 4f53cbfdd3..db0595e7bd 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/TestId.LegacyStrategy.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/TestId.LegacyStrategy.cs @@ -11,14 +11,14 @@ public partial class TestId : CLITestBase { private const string LegacyStrategyDll = "TestIdProject.LegacyStrategy"; - public void TestIdUniqueness_DataRowArray_LegacyStrategy() + public async Task TestIdUniqueness_DataRowArray_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -33,14 +33,14 @@ public void TestIdUniqueness_DataRowArray_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DataRowString_LegacyStrategy() + public async Task TestIdUniqueness_DataRowString_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowStringTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -56,14 +56,14 @@ public void TestIdUniqueness_DataRowString_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DynamicDataArrays_LegacyStrategy() + public async Task TestIdUniqueness_DynamicDataArrays_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -78,14 +78,14 @@ public void TestIdUniqueness_DynamicDataArrays_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DynamicDataTuple_LegacyStrategy() + public async Task TestIdUniqueness_DynamicDataTuple_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -99,14 +99,14 @@ public void TestIdUniqueness_DynamicDataTuple_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_DynamicDataGenericCollections_LegacyStrategy() + public async Task TestIdUniqueness_DynamicDataGenericCollections_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DynamicDataGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -122,14 +122,14 @@ public void TestIdUniqueness_DynamicDataGenericCollections_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceArrays_LegacyStrategy() + public async Task TestIdUniqueness_TestDataSourceArrays_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceArraysTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -144,14 +144,14 @@ public void TestIdUniqueness_TestDataSourceArrays_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceTuples_LegacyStrategy() + public async Task TestIdUniqueness_TestDataSourceTuples_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceTuplesTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); @@ -165,14 +165,14 @@ public void TestIdUniqueness_TestDataSourceTuples_LegacyStrategy() testResults.Select(x => x.TestCase.Id.ToString()).Distinct().Should().ContainSingle(); } - public void TestIdUniqueness_TestDataSourceGenericCollections_LegacyStrategy() + public async Task TestIdUniqueness_TestDataSourceGenericCollections_LegacyStrategy() { // Arrange string assemblyPath = GetAssetFullPath(LegacyStrategyDll); // Act System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~TestDataSourceGenericCollectionsTests"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + System.Collections.Immutable.ImmutableArray testResults = await RunTestsAsync(testCases); // Assert VerifyE2E.FailedTestCount(testResults, 0); diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs b/test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs index 31d9057b6f..94367a837a 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs @@ -33,12 +33,12 @@ internal static ImmutableArray DiscoverTests(string assemblyPath, stri return sink.DiscoveredTests; } - internal static ImmutableArray RunTests(IEnumerable testCases) + internal static async Task> RunTestsAsync(IEnumerable testCases) { var testExecutionManager = new TestExecutionManager(); var frameworkHandle = new InternalFrameworkHandle(); - testExecutionManager.ExecuteTests(testCases, null, frameworkHandle, false); + await testExecutionManager.ExecuteTestsAsync(testCases, null, frameworkHandle, false); return frameworkHandle.GetFlattenedTestResults(); } diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs index f00010e440..0e70dfcfab 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs @@ -61,15 +61,11 @@ public TestExecutionManagerTests() _testExecutionManager = new TestExecutionManager( new EnvironmentWrapper(), - action => Task.Factory.StartNew( - () => - { - _enqueuedParallelTestsCount++; - action(); - }, - CancellationToken.None, - TaskCreationOptions.LongRunning, - TaskScheduler.Default)); + task => + { + _enqueuedParallelTestsCount++; + return task(); + }); } protected override void Dispose(bool disposing) @@ -1137,7 +1133,11 @@ internal class TestableTestExecutionManager : TestExecutionManager { internal Action, IRunContext, IFrameworkHandle, bool> ExecuteTestsWrapper { get; set; } - internal override void ExecuteTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) => ExecuteTestsWrapper?.Invoke(tests, runContext, frameworkHandle, isDeploymentDone); + internal override Task ExecuteTestsAsync(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) + { + ExecuteTestsWrapper?.Invoke(tests, runContext, frameworkHandle, isDeploymentDone); + return Task.CompletedTask; + } internal override UnitTestDiscoverer GetUnitTestDiscoverer() => new TestableUnitTestDiscoverer(); } diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/UnitTestRunnerTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/UnitTestRunnerTests.cs index 52524eabf9..d9941fcd5a 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/UnitTestRunnerTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/UnitTestRunnerTests.cs @@ -80,23 +80,23 @@ public void ConstructorShouldPopulateSettings() #region RunSingleTest tests - public void RunSingleTestShouldThrowIfTestMethodIsNull() => - VerifyThrows(() => _unitTestRunner.RunSingleTest(null, null, null)); + public async Task RunSingleTestShouldThrowIfTestMethodIsNull() => + await VerifyThrowsAsync(async () => await _unitTestRunner.RunSingleTestAsync(null, null, null)); - public void RunSingleTestShouldThrowIfTestRunParametersIsNull() + public async Task RunSingleTestShouldThrowIfTestRunParametersIsNull() { var testMethod = new TestMethod("M", "C", "A", isAsync: false); - VerifyThrows(() => _unitTestRunner.RunSingleTest(testMethod, null, null)); + await VerifyThrowsAsync(async () => await _unitTestRunner.RunSingleTestAsync(testMethod, null, null)); } - public void RunSingleTestShouldReturnTestResultIndicateATestNotFoundIfTestMethodCannotBeFound() + public async Task RunSingleTestShouldReturnTestResultIndicateATestNotFoundIfTestMethodCannotBeFound() { var testMethod = new TestMethod("M", "C", "A", isAsync: false); _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -104,7 +104,7 @@ public void RunSingleTestShouldReturnTestResultIndicateATestNotFoundIfTestMethod Verify(results[0].ErrorMessage == "Test method M was not found."); } - public void RunSingleTestShouldReturnTestResultIndicatingNotRunnableTestIfTestMethodCannotBeRun() + public async Task RunSingleTestShouldReturnTestResultIndicatingNotRunnableTestIfTestMethodCannotBeRun() { Type type = typeof(TypeCacheTests.DummyTestClassWithTestMethods); MethodInfo methodInfo = type.GetMethod("TestMethodWithNullCustomPropertyName"); @@ -113,7 +113,7 @@ public void RunSingleTestShouldReturnTestResultIndicatingNotRunnableTestIfTestMe _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); string expectedMessage = string.Format( CultureInfo.InvariantCulture, @@ -127,7 +127,7 @@ public void RunSingleTestShouldReturnTestResultIndicatingNotRunnableTestIfTestMe Verify(expectedMessage == results[0].ErrorMessage); } - public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsPresentOnTestClassAndHasMessage() + public async Task ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsPresentOnTestClassAndHasMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreClassWithMessage); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -136,7 +136,7 @@ public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsP _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -144,7 +144,7 @@ public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsP Verify(results[0].ErrorMessage == "IgnoreTestClassMessage"); } - public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsPresentOnTestClassButHasNoMessage() + public async Task ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsPresentOnTestClassButHasNoMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreClass); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -153,7 +153,7 @@ public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsP _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -161,7 +161,7 @@ public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsP Verify(results[0].ErrorMessage == string.Empty); } - public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIsPresentOnTestMethodAndHasMessage() + public async Task ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIsPresentOnTestMethodAndHasMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreTestWithMessage); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -170,7 +170,7 @@ public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIs _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -178,7 +178,7 @@ public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIs Verify(results[0].ErrorMessage == "IgnoreTestMessage"); } - public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsPresentOnTestMethodButHasNoMessage() + public async Task ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsPresentOnTestMethodButHasNoMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreTest); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -187,7 +187,7 @@ public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsP _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -195,7 +195,7 @@ public void ExecuteShouldSkipTestAndSkipFillingIgnoreMessageIfIgnoreAttributeIsP Verify(results[0].ErrorMessage == string.Empty); } - public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsPresentOnBothClassAndMethod() + public async Task ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsPresentOnBothClassAndMethod() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreClassAndIgnoreTestWithMessage); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -204,7 +204,7 @@ public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsP _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -212,7 +212,7 @@ public void ExecuteShouldSkipTestAndFillInClassIgnoreMessageIfIgnoreAttributeIsP Verify(results[0].ErrorMessage == "IgnoreTestClassMessage"); } - public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIsPresentOnBothClassAndMethodButClassHasNoMessage() + public async Task ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIsPresentOnBothClassAndMethodButClassHasNoMessage() { Type type = typeof(TypeCacheTests.DummyTestClassWithIgnoreClassWithNoMessageAndIgnoreTestWithMessage); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -221,7 +221,7 @@ public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIs _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -229,7 +229,7 @@ public void ExecuteShouldSkipTestAndFillInMethodIgnoreMessageIfIgnoreAttributeIs Verify(results[0].ErrorMessage == "IgnoreTestMessage"); } - public void RunSingleTestShouldReturnTestResultIndicatingFailureIfThereIsAnyTypeInspectionExceptionWhenInspectingTestMethod() + public async Task RunSingleTestShouldReturnTestResultIndicatingFailureIfThereIsAnyTypeInspectionExceptionWhenInspectingTestMethod() { Type type = typeof(TypeCacheTests.DummyTestClassWithTestMethods); var testMethod = new TestMethod("ImaginaryTestMethod", type.FullName, "A", isAsync: false); @@ -237,7 +237,7 @@ public void RunSingleTestShouldReturnTestResultIndicatingFailureIfThereIsAnyType _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); string expectedMessage = string.Format( CultureInfo.InvariantCulture, @@ -251,7 +251,7 @@ public void RunSingleTestShouldReturnTestResultIndicatingFailureIfThereIsAnyType Verify(expectedMessage == results[0].ErrorMessage); } - public void RunSingleTestShouldReturnTestResultsForAPassingTestMethod() + public async Task RunSingleTestShouldReturnTestResultsForAPassingTestMethod() { Type type = typeof(TypeCacheTests.DummyTestClassWithTestMethods); MethodInfo methodInfo = type.GetMethod("TestMethod"); @@ -260,7 +260,7 @@ public void RunSingleTestShouldReturnTestResultsForAPassingTestMethod() _testablePlatformServiceProvider.MockFileOperations.Setup(fo => fo.LoadAssembly("A", It.IsAny())) .Returns(Assembly.GetExecutingAssembly()); - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); @@ -268,7 +268,7 @@ public void RunSingleTestShouldReturnTestResultsForAPassingTestMethod() Verify(results[0].ErrorMessage is null); } - public void RunSingleTestShouldSetTestsAsInProgressInTestContext() + public async Task RunSingleTestShouldSetTestsAsInProgressInTestContext() { Type type = typeof(DummyTestClass); MethodInfo methodInfo = type.GetMethod("TestMethodToTestInProgress"); @@ -278,14 +278,14 @@ public void RunSingleTestShouldSetTestsAsInProgressInTestContext() .Returns(Assembly.GetExecutingAssembly()); // Asserting in the test method execution flow itself. - UnitTestResult[] results = _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + UnitTestResult[] results = await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(results is not null); Verify(results.Length == 1); Verify(results[0].Outcome == UnitTestOutcome.Passed); } - public void RunSingleTestShouldCallAssemblyInitializeAndClassInitializeMethodsInOrder() + public async Task RunSingleTestShouldCallAssemblyInitializeAndClassInitializeMethodsInOrder() { var mockReflectHelper = new Mock(); _unitTestRunner = new UnitTestRunner(new MSTestSettings(), Array.Empty(), null, mockReflectHelper.Object); @@ -304,7 +304,7 @@ public void RunSingleTestShouldCallAssemblyInitializeAndClassInitializeMethodsIn DummyTestClassWithInitializeMethods.AssemblyInitializeMethodBody = () => validator <<= 2; DummyTestClassWithInitializeMethods.ClassInitializeMethodBody = () => validator >>= 2; - _unitTestRunner.RunSingleTest(testMethod, _testRunParameters, null); + await _unitTestRunner.RunSingleTestAsync(testMethod, _testRunParameters, null); Verify(validator == 1); }