diff --git a/source/Halibut.Tests/Diagnostics/ExceptionReturnedByHalibutProxyExtensionMethodFixture.cs b/source/Halibut.Tests/Diagnostics/ExceptionReturnedByHalibutProxyExtensionMethodFixture.cs index 8a4e48f50..a7d49c1fb 100644 --- a/source/Halibut.Tests/Diagnostics/ExceptionReturnedByHalibutProxyExtensionMethodFixture.cs +++ b/source/Halibut.Tests/Diagnostics/ExceptionReturnedByHalibutProxyExtensionMethodFixture.cs @@ -11,7 +11,7 @@ using Halibut.Tests.TestServices; using Halibut.Tests.TestServices.Async; using Halibut.TestUtils.Contracts; -using NUnit.Framework; +using Xunit; namespace Halibut.Tests.Diagnostics { @@ -19,7 +19,7 @@ public static class ExceptionReturnedByHalibutProxyExtensionMethodFixture { public class WhenGivenA { - [Test] + [Fact] public void MethodNotFoundHalibutClientException_ItIsNotANetworkError() { new MethodNotFoundHalibutClientException("").IsNetworkError() @@ -27,7 +27,7 @@ public void MethodNotFoundHalibutClientException_ItIsNotANetworkError() .Be(HalibutNetworkExceptionType.NotANetworkError); } - [Test] + [Fact] public void ServiceNotFoundHalibutClientException_ItIsNotANetworkError() { new ServiceNotFoundHalibutClientException("").IsNetworkError() @@ -35,7 +35,7 @@ public void ServiceNotFoundHalibutClientException_ItIsNotANetworkError() .Be(HalibutNetworkExceptionType.NotANetworkError); } - [Test] + [Fact] public void AmbiguousMethodMatchHalibutClientException_ItIsNotANetworkError() { new AmbiguousMethodMatchHalibutClientException("").IsNetworkError() @@ -44,9 +44,10 @@ public void AmbiguousMethodMatchHalibutClientException_ItIsNotANetworkError() } } - public class WhenTheHalibutProxyThrowsAnException : BaseTest + public class WhenTheHalibutProxyThrowsAnException : BaseTestXUnit { - [LatestClientAndLatestServiceTestCases(testNetworkConditions:false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions:false)] public async Task WhenTheConnectionTerminatesWaitingForAResponse(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -74,8 +75,9 @@ public async Task WhenTheConnectionTerminatesWaitingForAResponse(ClientAndServic because: "This isn't the best message, really the connection was closed before we got the data we were expecting resulting in us reading past the end of the stream"); } } - - [LatestClientAndLatestServiceTestCases(testNetworkConditions:false, + + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions:false, testWebSocket:false // Since websockets do not timeout )] public async Task WhenTheConnectionPausesWaitingForAResponse(ClientAndServiceTestCase clientAndServiceTestCase) @@ -101,8 +103,8 @@ public async Task WhenTheConnectionPausesWaitingForAResponse(ClientAndServiceTes } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false, testListening:false, testWebSocket: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false, testListening:false, testWebSocket: false)] public async Task BecauseThePollingRequestWasNotCollected(ClientAndServiceTestCase clientAndServiceTestCase) { var services = new DelegateServiceFactory(); @@ -121,8 +123,8 @@ public async Task BecauseThePollingRequestWasNotCollected(ClientAndServiceTestCa } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false, testPolling: false, testWebSocket: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false, testPolling: false, testWebSocket: false)] public async Task BecauseTheListeningTentacleIsNotResponding(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -138,8 +140,8 @@ public async Task BecauseTheListeningTentacleIsNotResponding(ClientAndServiceTes } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false, testWebSocket: false, testPolling: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false, testWebSocket: false, testPolling: false)] public async Task BecauseTheProxyIsNotResponding_TheExceptionShouldBeANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -161,8 +163,8 @@ public async Task BecauseTheProxyIsNotResponding_TheExceptionShouldBeANetworkErr } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false, testPolling: false, testWebSocket: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false, testPolling: false, testWebSocket: false)] public async Task BecauseOfAInvalidCertificateException_WhenConnectingToListening_ItIsNotANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -181,8 +183,8 @@ public async Task BecauseOfAInvalidCertificateException_WhenConnectingToListenin } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false)] public async Task BecauseTheDataStreamHadAnErrorOpeningTheFileWithFileStream_WhenSending_ItIsNotANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -208,8 +210,8 @@ public async Task BecauseTheDataStreamHadAnErrorOpeningTheFileWithFileStream_Whe } } - [Test] - [LatestClientAndLatestServiceTestCases(testNetworkConditions: false)] + [Theory] + [LatestClientAndLatestServiceTestCasesXUnit(testNetworkConditions: false)] public async Task BecauseTheDataStreamThrowAFileNotFoundException_WhenSending_ItIsNotANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() @@ -222,7 +224,7 @@ public async Task BecauseTheDataStreamThrowAFileNotFoundException_WhenSending_It _ => throw new FileNotFoundException(), async (_, _) => { - await Task.CompletedTask.ConfigureAwait(false); + await Task.CompletedTask; throw new FileNotFoundException(); }); @@ -234,8 +236,8 @@ public async Task BecauseTheDataStreamThrowAFileNotFoundException_WhenSending_It } } - [Test] - [LatestAndPreviousClientAndServiceVersionsTestCases(testNetworkConditions: false)] + [Theory] + [LatestAndPreviousClientAndServiceVersionsTestCasesXUnit(testNetworkConditions: false)] public async Task BecauseTheServiceThrowAnException_ItIsNotANetworkError(ClientAndServiceTestCase clientAndServiceTestCase) { await using (var clientAndService = await clientAndServiceTestCase.CreateTestCaseBuilder() diff --git a/source/Halibut.Tests/Halibut.Tests.csproj b/source/Halibut.Tests/Halibut.Tests.csproj index 4bcd47f28..b790a6f9a 100644 --- a/source/Halibut.Tests/Halibut.Tests.csproj +++ b/source/Halibut.Tests/Halibut.Tests.csproj @@ -47,6 +47,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/source/Halibut.Tests/Properties/AssemblyInfo.cs b/source/Halibut.Tests/Properties/AssemblyInfo.cs index 9d65eada2..802e380e1 100644 --- a/source/Halibut.Tests/Properties/AssemblyInfo.cs +++ b/source/Halibut.Tests/Properties/AssemblyInfo.cs @@ -1,7 +1,9 @@ using System; using System.Reflection; using Halibut.Tests.Support.TestAttributes; +using Halibut.Tests.Util; using NUnit.Framework; +using Xunit; // Information about this assembly is defined by the following attributes. // Change them to the values specific to your project. @@ -11,3 +13,5 @@ [assembly: FixtureLifeCycle(LifeCycle.InstancePerTestCase)] [assembly: TestTimeout] [assembly: CustomLevelOfParallelism] +[assembly: TestFramework("Halibut.Tests.Util." + nameof(ParallelTestFramework), "Halibut.Tests")] +[assembly: CollectionBehavior(MaxParallelThreads = 2)] \ No newline at end of file diff --git a/source/Halibut.Tests/Support/BackwardsCompatibility/PreviousVersions.cs b/source/Halibut.Tests/Support/BackwardsCompatibility/PreviousVersions.cs index 525a4261b..83c205eda 100644 --- a/source/Halibut.Tests/Support/BackwardsCompatibility/PreviousVersions.cs +++ b/source/Halibut.Tests/Support/BackwardsCompatibility/PreviousVersions.cs @@ -1,4 +1,5 @@ using System; +using Xunit.Abstractions; namespace Halibut.Tests.Support.BackwardsCompatibility { @@ -22,11 +23,15 @@ public static class PreviousVersions #endif } - public class HalibutVersion : IEquatable + public class HalibutVersion : IEquatable, IXunitSerializable { - readonly Version pollingVersion; - readonly Version listeningVersion; - readonly Version pollingOverWebSocketVersion; + Version pollingVersion; + Version listeningVersion; + Version pollingOverWebSocketVersion; + + public HalibutVersion() + { + } internal HalibutVersion( Version version, @@ -77,6 +82,20 @@ public override int GetHashCode() return hashCode; } } + + public void Deserialize(IXunitSerializationInfo info) + { + pollingVersion = Version.Parse(info.GetValue(nameof(pollingVersion))); + listeningVersion = Version.Parse(info.GetValue(nameof(listeningVersion))); + pollingOverWebSocketVersion = Version.Parse(info.GetValue(nameof(pollingOverWebSocketVersion))); + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(pollingVersion), pollingVersion.ToString()); + info.AddValue(nameof(listeningVersion), listeningVersion.ToString()); + info.AddValue(nameof(pollingOverWebSocketVersion), pollingOverWebSocketVersion.ToString()); + } } public class HalibutVersions diff --git a/source/Halibut.Tests/Support/TestAttributes/HalibutTestCaseSourceAttribute.cs b/source/Halibut.Tests/Support/TestAttributes/HalibutTestCaseSourceAttribute.cs index 6d068625a..47b0847ee 100644 --- a/source/Halibut.Tests/Support/TestAttributes/HalibutTestCaseSourceAttribute.cs +++ b/source/Halibut.Tests/Support/TestAttributes/HalibutTestCaseSourceAttribute.cs @@ -162,6 +162,18 @@ private IEnumerable GetTestCasesFor(IMethodInfo method) { parms.Properties.Add(PropertyNames.Category, testCase.ServiceConnectionType.ToString()); parms.Properties.Add(PropertyNames.Category, testCase.SyncOrAsync.ToString()); + } + else if (item is IEnumerable enumerableItem) + { + foreach (var itemFromEnumerable in enumerableItem) + { + if (itemFromEnumerable is ClientAndServiceTestCase testCaseFromEnumerable) + { + parms.Properties.Add(PropertyNames.Category, testCaseFromEnumerable.ServiceConnectionType.ToString()); + parms.Properties.Add(PropertyNames.Category, testCaseFromEnumerable.SyncOrAsync.ToString()); + break; + } + } } data.Add(parms); diff --git a/source/Halibut.Tests/Support/TestAttributes/LatestAndPreviousClientAndServiceVersionsTestCasesAttribute.cs b/source/Halibut.Tests/Support/TestAttributes/LatestAndPreviousClientAndServiceVersionsTestCasesAttribute.cs index 4402ddd5f..70064109b 100644 --- a/source/Halibut.Tests/Support/TestAttributes/LatestAndPreviousClientAndServiceVersionsTestCasesAttribute.cs +++ b/source/Halibut.Tests/Support/TestAttributes/LatestAndPreviousClientAndServiceVersionsTestCasesAttribute.cs @@ -1,9 +1,12 @@ using System; -using System.Collections; +using System.Collections.Generic; using System.Linq; +using System.Reflection; using Halibut.Tests.Support.BackwardsCompatibility; using Halibut.Tests.Support.TestCases; using Halibut.Util; +using Xunit.Sdk; +using static Halibut.Tests.Support.TestAttributes.LatestAndPreviousClientAndServiceVersionsTestCasesAttribute; namespace Halibut.Tests.Support.TestAttributes { @@ -17,17 +20,18 @@ public LatestAndPreviousClientAndServiceVersionsTestCasesAttribute( bool testPolling = true, bool testAsyncAndSyncClients = true, bool testAsyncServicesAsWell = false // False means only the sync service will be tested. - ) : + ) : base( typeof(LatestAndPreviousClientAndServiceVersionsTestCases), nameof(LatestAndPreviousClientAndServiceVersionsTestCases.GetEnumerator), - new object[] { testWebSocket, testNetworkConditions, testListening, testPolling, testAsyncAndSyncClients, testAsyncServicesAsWell}) + new object[] { testWebSocket, testNetworkConditions, testListening, testPolling, testAsyncAndSyncClients, testAsyncServicesAsWell }) { } - - static class LatestAndPreviousClientAndServiceVersionsTestCases + + public static class LatestAndPreviousClientAndServiceVersionsTestCases { - public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testAsyncAndSyncClients, bool testAsyncServicesAsWell) + //TODO: @server-at-scale - When NUnit is removed, remove this class, and make this method the body of LatestAndPreviousClientAndServiceVersionsTestCasesXUnitAttribute.GetData + public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testAsyncAndSyncClients, bool testAsyncServicesAsWell) { var serviceConnectionTypes = ServiceConnectionTypes.All.ToList(); @@ -51,15 +55,16 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond { clientProxyTypesToTest = ForceClientProxyTypeValues.All; } - + var serviceAsyncHalibutFeatureTestCases = AsyncHalibutFeatureValues.All().ToList(); if (!testAsyncServicesAsWell) { serviceAsyncHalibutFeatureTestCases.Remove(AsyncHalibutFeature.Enabled); } - + var builder = new ClientAndServiceTestCasesBuilder( - new[] { + new[] + { ClientAndServiceTestVersion.Latest(), ClientAndServiceTestVersion.ClientOfVersion(PreviousVersions.v5_0_236_Used_In_Tentacle_6_3_417.ClientVersion), ClientAndServiceTestVersion.ServiceOfVersion(PreviousVersions.v5_0_236_Used_In_Tentacle_6_3_417.ServiceVersion), @@ -69,10 +74,53 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond testNetworkConditions ? NetworkConditionTestCase.All : new[] { NetworkConditionTestCase.NetworkConditionPerfect }, clientProxyTypesToTest, serviceAsyncHalibutFeatureTestCases - ); + ); return builder.Build(); } } } + + //TODO: @server-at-scale - When NUnit is removed, replace the above LatestAndPreviousClientAndServiceVersionsTestCasesAttribute with this. + [AttributeUsage(AttributeTargets.Method, AllowMultiple = true, Inherited = false)] + public class LatestAndPreviousClientAndServiceVersionsTestCasesXUnitAttribute : DataAttribute + { + readonly bool testWebSocket; + readonly bool testNetworkConditions; + readonly bool testListening; + readonly bool testPolling; + readonly bool testAsyncAndSyncClients; + readonly bool testAsyncServicesAsWell; + + public LatestAndPreviousClientAndServiceVersionsTestCasesXUnitAttribute( + bool testWebSocket = true, + bool testNetworkConditions = true, + bool testListening = true, + bool testPolling = true, + bool testAsyncAndSyncClients = true, + bool testAsyncServicesAsWell = false // False means only the sync service will be tested. + ) + { + this.testWebSocket = testWebSocket; + this.testNetworkConditions = testNetworkConditions; + this.testListening = testListening; + this.testPolling = testPolling; + this.testAsyncAndSyncClients = testAsyncAndSyncClients; + this.testAsyncServicesAsWell = testAsyncServicesAsWell; + } + + public override IEnumerable GetData(MethodInfo testMethod) + { + //TODO: @server-at-scale - When NUnit is removed, move LatestClientAndLatestServiceTestCases.GetEnumerator into here. + + return LatestAndPreviousClientAndServiceVersionsTestCases.GetEnumerator( + testWebSocket, + testNetworkConditions, + testListening, + testPolling, + testAsyncAndSyncClients, + testAsyncServicesAsWell); + } + } + } diff --git a/source/Halibut.Tests/Support/TestAttributes/LatestClientAndLatestServiceTestCasesAttribute.cs b/source/Halibut.Tests/Support/TestAttributes/LatestClientAndLatestServiceTestCasesAttribute.cs index 0a7c36fce..70a5a3fca 100644 --- a/source/Halibut.Tests/Support/TestAttributes/LatestClientAndLatestServiceTestCasesAttribute.cs +++ b/source/Halibut.Tests/Support/TestAttributes/LatestClientAndLatestServiceTestCasesAttribute.cs @@ -1,9 +1,11 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Reflection; using Halibut.Tests.Support.TestCases; using Halibut.Util; +using Xunit.Sdk; +using static Halibut.Tests.Support.TestAttributes.LatestClientAndLatestServiceTestCasesAttribute; namespace Halibut.Tests.Support.TestAttributes { @@ -24,17 +26,18 @@ public LatestClientAndLatestServiceTestCasesAttribute( bool testAsyncServicesAsWell = false, // False means only the sync service will be tested. bool testSyncService = true, params object[] additionalParameters - ) : + ) : base( typeof(LatestClientAndLatestServiceTestCases), nameof(LatestClientAndLatestServiceTestCases.GetEnumerator), new object[] { testWebSocket, testNetworkConditions, testListening, testPolling, testSyncClients, testAsyncClients, testAsyncServicesAsWell, testSyncService, additionalParameters }) { } - - static class LatestClientAndLatestServiceTestCases + + public static class LatestClientAndLatestServiceTestCases { - public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testSyncClients, bool testAsyncClients, bool testAsyncServicesAsWell, bool testSyncService, object[] additionalParameters) + //TODO: @server-at-scale - When NUnit is removed, remove this class, and make this method the body of LatestClientAndLatestServiceTestCasesXUnitAttribute.GetData + public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testSyncClients, bool testAsyncClients, bool testAsyncServicesAsWell, bool testSyncService, object[] additionalParameters) { var serviceConnectionTypes = ServiceConnectionTypes.All.ToList(); @@ -52,9 +55,9 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond { serviceConnectionTypes.Remove(ServiceConnectionType.Polling); } - + List clientProxyTypesToTest = new(); - + if (testAsyncClients) { clientProxyTypesToTest.Add(ForceClientProxyType.AsyncClient); @@ -64,7 +67,7 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond clientProxyTypesToTest.Add(ForceClientProxyType.SyncClient); } } - + var serviceAsyncHalibutFeatureTestCases = AsyncHalibutFeatureValues.All().ToList(); if (!testAsyncServicesAsWell) { @@ -83,7 +86,7 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond clientProxyTypesToTest, serviceAsyncHalibutFeatureTestCases ); - + foreach (var clientAndServiceTestCase in builder.Build()) { if (additionalParameters.Any()) @@ -98,16 +101,67 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond } } - static object[] CombineTestCaseWithAdditionalParameters(ClientAndServiceTestCase clientAndServiceTestCase, object[] additionalParameters) + static object[] CombineTestCaseWithAdditionalParameters(object[] clientAndServiceTestCase, object[] additionalParameters) { - var parameters = new object[1 + additionalParameters.Length]; - - parameters[0] = clientAndServiceTestCase; - - Array.Copy(additionalParameters, 0, parameters, 1, additionalParameters.Length); + var parameters = new object[clientAndServiceTestCase.Length + additionalParameters.Length]; + + Array.Copy(clientAndServiceTestCase, 0, parameters, 0, clientAndServiceTestCase.Length); + Array.Copy(additionalParameters, 0, parameters, clientAndServiceTestCase.Length, additionalParameters.Length); return parameters; } } } + + //TODO: @server-at-scale - When NUnit is removed, replace the above LatestClientAndLatestServiceTestCasesAttribute with this. + public class LatestClientAndLatestServiceTestCasesXUnitAttribute : DataAttribute + { + readonly bool testWebSocket; + readonly bool testNetworkConditions; + readonly bool testListening; + readonly bool testPolling; + readonly bool testSyncClients; + readonly bool testAsyncClients; + readonly bool testAsyncServicesAsWell; + readonly bool testSyncService; + readonly object[] additionalParameters; + + public LatestClientAndLatestServiceTestCasesXUnitAttribute( + bool testWebSocket = true, + bool testNetworkConditions = true, + bool testListening = true, + bool testPolling = true, + bool testSyncClients = true, + bool testAsyncClients = true, + bool testAsyncServicesAsWell = false, // False means only the sync service will be tested. + bool testSyncService = true, + params object[] additionalParameters) + { + this.testWebSocket = testWebSocket; + this.testNetworkConditions = testNetworkConditions; + this.testListening = testListening; + this.testPolling = testPolling; + this.testSyncClients = testSyncClients; + this.testAsyncClients = testAsyncClients; + this.testAsyncServicesAsWell = testAsyncServicesAsWell; + this.testSyncService = testSyncService; + this.additionalParameters = additionalParameters; + } + + public override IEnumerable GetData(MethodInfo testMethod) + { + //TODO: @server-at-scale - When NUnit is removed, move LatestClientAndLatestServiceTestCases.GetEnumerator into here. + + return LatestClientAndLatestServiceTestCases.GetEnumerator( + testWebSocket, + testNetworkConditions, + testListening, + testPolling, + testSyncClients, + testAsyncClients, + testAsyncServicesAsWell, + testSyncService, + additionalParameters); + } + } } diff --git a/source/Halibut.Tests/Support/TestAttributes/LatestClientAndPreviousServiceVersionsTestCasesAttribute.cs b/source/Halibut.Tests/Support/TestAttributes/LatestClientAndPreviousServiceVersionsTestCasesAttribute.cs index b832e0ad0..cc7fdfed9 100644 --- a/source/Halibut.Tests/Support/TestAttributes/LatestClientAndPreviousServiceVersionsTestCasesAttribute.cs +++ b/source/Halibut.Tests/Support/TestAttributes/LatestClientAndPreviousServiceVersionsTestCasesAttribute.cs @@ -1,10 +1,12 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Reflection; using Halibut.Tests.Support.BackwardsCompatibility; using Halibut.Tests.Support.TestCases; using Halibut.Util; +using Xunit.Sdk; +using static Halibut.Tests.Support.TestAttributes.LatestClientAndPreviousServiceVersionsTestCasesAttribute; namespace Halibut.Tests.Support.TestAttributes { @@ -26,9 +28,10 @@ public LatestClientAndPreviousServiceVersionsTestCasesAttribute(bool testWebSock { } - static class LatestClientAndPreviousServiceVersionsTestCases + public static class LatestClientAndPreviousServiceVersionsTestCases { - public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testAsyncClients, bool testSyncClients, bool testAsyncServicesAsWell) + //TODO: @server-at-scale - When NUnit is removed, remove this class, and make this method the body of LatestClientAndPreviousServiceVersionsTestCasesXUnitAttribute.GetData + public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkConditions, bool testListening, bool testPolling, bool testAsyncClients, bool testSyncClients, bool testAsyncServicesAsWell) { var serviceConnectionTypes = ServiceConnectionTypes.All.ToList(); @@ -81,4 +84,50 @@ public static IEnumerable GetEnumerator(bool testWebSocket, bool testNetworkCond } } } + + //TODO: @server-at-scale - When NUnit is removed, replace the above LatestClientAndPreviousServiceVersionsTestCasesAttribute with this. + [AttributeUsage(AttributeTargets.Method, AllowMultiple = true, Inherited = false)] + public class LatestClientAndPreviousServiceVersionsTestCasesXUnitAttribute : DataAttribute + { + readonly bool testWebSocket; + readonly bool testNetworkConditions; + readonly bool testListening; + readonly bool testPolling; + readonly bool testAsyncClients; + readonly bool testSyncClients; + readonly bool testAsyncServicesAsWell; + + public LatestClientAndPreviousServiceVersionsTestCasesXUnitAttribute( + bool testWebSocket = true, + bool testNetworkConditions = true, + bool testListening = true, + bool testPolling = true, + bool testAsyncClients = true, + bool testSyncClients = true, + bool testAsyncServicesAsWell = false // False means only the sync service will be tested. + ) + { + this.testWebSocket = testWebSocket; + this.testNetworkConditions = testNetworkConditions; + this.testListening = testListening; + this.testPolling = testPolling; + this.testAsyncClients = testAsyncClients; + this.testSyncClients = testSyncClients; + this.testAsyncServicesAsWell = testAsyncServicesAsWell; + } + + public override IEnumerable GetData(MethodInfo testMethod) + { + //TODO: @server-at-scale - When NUnit is removed, move LatestClientAndLatestServiceTestCases.GetEnumerator into here. + + return LatestClientAndPreviousServiceVersionsTestCases.GetEnumerator( + testWebSocket, + testNetworkConditions, + testListening, + testPolling, + testAsyncClients, + testSyncClients, + testAsyncServicesAsWell); + } + } } diff --git a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs index 40204c500..27d53b982 100644 --- a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs +++ b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCase.cs @@ -1,32 +1,37 @@ using System; -using System.Collections.Generic; using System.Text; using Halibut.Tests.Support.TestAttributes; using Halibut.Util; +using Xunit.Abstractions; namespace Halibut.Tests.Support.TestCases { - public class ClientAndServiceTestCase + public class ClientAndServiceTestCase : IXunitSerializable { - public ClientAndServiceTestVersion ClientAndServiceTestVersion { get; } + public ClientAndServiceTestVersion ClientAndServiceTestVersion { get; private set; } - public NetworkConditionTestCase NetworkConditionTestCase { get; } + public NetworkConditionTestCase NetworkConditionTestCase { get; private set; } - public ServiceConnectionType ServiceConnectionType { get; } + public ServiceConnectionType ServiceConnectionType { get; private set; } - public AsyncHalibutFeature ServiceAsyncHalibutFeature { get; } + public AsyncHalibutFeature ServiceAsyncHalibutFeature { get; private set; } /// /// If running a test which wants to make the same or similar calls multiple time, this has a "good" /// number of times to do that. It takes care of generally not picking such a high number the tests /// don't take a long time. /// - public int RecommendedIterations { get; } + public int RecommendedIterations { get; private set; } - public ForceClientProxyType? ForceClientProxyType { get; } + public ForceClientProxyType? ForceClientProxyType { get; private set; } public SyncOrAsync SyncOrAsync => ForceClientProxyType.ToSyncOrAsync(); + + public ClientAndServiceTestCase() + { + } - public ClientAndServiceTestCase(ServiceConnectionType serviceConnectionType, + public ClientAndServiceTestCase( + ServiceConnectionType serviceConnectionType, NetworkConditionTestCase networkConditionTestCase, int recommendedIterations, ClientAndServiceTestVersion clientAndServiceTestVersion, @@ -114,5 +119,25 @@ public override int GetHashCode() { return ToString().GetHashCode(); } + + public void Deserialize(IXunitSerializationInfo info) + { + ClientAndServiceTestVersion = info.GetValue(nameof(ClientAndServiceTestVersion)); + NetworkConditionTestCase = info.GetValue(nameof(NetworkConditionTestCase)); + ServiceConnectionType = info.GetValue(nameof(ServiceConnectionType)); + ServiceAsyncHalibutFeature = info.GetValue(nameof(ServiceAsyncHalibutFeature)); + RecommendedIterations = info.GetValue(nameof(RecommendedIterations)); + ForceClientProxyType = info.GetValue(nameof(ForceClientProxyType)); + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(ClientAndServiceTestVersion), ClientAndServiceTestVersion); + info.AddValue(nameof(NetworkConditionTestCase), NetworkConditionTestCase); + info.AddValue(nameof(ServiceConnectionType), ServiceConnectionType); + info.AddValue(nameof(ServiceAsyncHalibutFeature), ServiceAsyncHalibutFeature); + info.AddValue(nameof(RecommendedIterations), RecommendedIterations); + info.AddValue(nameof(ForceClientProxyType), ForceClientProxyType); + } } } diff --git a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCasesBuilder.cs b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCasesBuilder.cs index 5fa4ed5bc..9e7eb7149 100644 --- a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCasesBuilder.cs +++ b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestCasesBuilder.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using Halibut.Tests.Util; @@ -26,16 +27,16 @@ public ClientAndServiceTestCasesBuilder( this.networkConditionTestCases = networkConditionTestCases.Distinct().ToArray(); this.forceClientProxyTypes = forceClientProxyTypes.Distinct().ToArray(); } - - public IEnumerable Build() + + public IEnumerable Build() { - return BuildDistinct(); + return BuildTestCases() + .Distinct() + .Select(tc => new object[]{tc}); } - List BuildDistinct() + IEnumerable BuildTestCases() { - var cases = new List(); - foreach (var clientServiceTestVersion in clientServiceTestVersions) { foreach (var serviceConnectionType in serviceConnectionTypes) @@ -59,19 +60,19 @@ List BuildDistinct() if (!forceClientProxyTypes.Any()) { - cases.Add(new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, null, serviceAsyncHalibutFeatureTestCase)); + yield return new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, null, serviceAsyncHalibutFeatureTestCase); } else { if (clientServiceTestVersion.IsPreviousClient()) { - cases.Add(new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, null, serviceAsyncHalibutFeatureTestCase)); + yield return new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, null, serviceAsyncHalibutFeatureTestCase); } else { foreach (var forceClientProxyType in forceClientProxyTypes) { - cases.Add(new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, forceClientProxyType, serviceAsyncHalibutFeatureTestCase)); + yield return new ClientAndServiceTestCase(serviceConnectionType, networkConditionTestCase, recommendedIterations, clientServiceTestVersion, forceClientProxyType, serviceAsyncHalibutFeatureTestCase); } } } @@ -79,10 +80,6 @@ List BuildDistinct() } } } - - var distinct = cases.Distinct().ToList(); - - return distinct; } } } diff --git a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestVersion.cs b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestVersion.cs index 5668cbe30..2aebd3839 100644 --- a/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestVersion.cs +++ b/source/Halibut.Tests/Support/TestCases/ClientAndServiceTestVersion.cs @@ -1,18 +1,23 @@ using System; using Halibut.Tests.Support.BackwardsCompatibility; +using Xunit.Abstractions; namespace Halibut.Tests.Support.TestCases { /// /// Defines what version of the Client and the Service that will be used in a test. /// - public class ClientAndServiceTestVersion : IEquatable + public class ClientAndServiceTestVersion : IEquatable, IXunitSerializable { // null means latest. - public HalibutVersion? ClientVersion { get; } + public HalibutVersion? ClientVersion { get; private set; } // null means latest. - public HalibutVersion? ServiceVersion { get; } + public HalibutVersion? ServiceVersion { get; private set; } + + public ClientAndServiceTestVersion() + { + } ClientAndServiceTestVersion(HalibutVersion? clientVersion, HalibutVersion? serviceVersion) { @@ -131,6 +136,18 @@ public override int GetHashCode() return hashCode; } } + + public void Deserialize(IXunitSerializationInfo info) + { + ClientVersion = info.GetValue(nameof(ClientVersion)); + ServiceVersion = info.GetValue(nameof(ServiceVersion)); + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(ClientVersion), ClientVersion); + info.AddValue(nameof(ServiceVersion), ServiceVersion); + } } public class ClientAndServiceBuilderFactory diff --git a/source/Halibut.Tests/Support/TestCases/NetworkConditionTestCase.cs b/source/Halibut.Tests/Support/TestCases/NetworkConditionTestCase.cs index f80e2b84d..7f5c11859 100644 --- a/source/Halibut.Tests/Support/TestCases/NetworkConditionTestCase.cs +++ b/source/Halibut.Tests/Support/TestCases/NetworkConditionTestCase.cs @@ -1,10 +1,11 @@ using System; using Octopus.TestPortForwarder; using Serilog; +using Xunit.Abstractions; namespace Halibut.Tests.Support.TestCases { - public class NetworkConditionTestCase + public class NetworkConditionTestCase : IXunitSerializable { public static NetworkConditionTestCase[] All => new[] { @@ -16,17 +17,17 @@ public class NetworkConditionTestCase }; public static NetworkConditionTestCase NetworkConditionPerfect = - new (null, + new (0, "Perfect", "Perfect"); public static NetworkConditionTestCase NetworkCondition20MsLatency = - new ((i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).Build(), + new (1, "20ms SendDelay", "20msLatency"); public static NetworkConditionTestCase NetworkCondition20MsLatencyWithLastByteArrivingLate = - new ((i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).WithNumberOfBytesToDelaySending(1).Build(), + new (2, "20ms SendDelay last byte arrives late", "20msLatency&LastByteLate"); @@ -38,15 +39,36 @@ public class NetworkConditionTestCase // new ((i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).WithNumberOfBytesToDelaySending(3).Build(), // "20ms send delay with last 3 bytes arriving late"); - public Func? PortForwarderFactory { get; } + //TODO: Make nicer. + int factory; + public Func? PortForwarderFactory + { + get + { + switch (factory) + { + case 1: + return (i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).Build(); + case 2: + return (i, logger) => PortForwarderBuilder.ForwardingToLocalPort(i, logger).WithSendDelay(TimeSpan.FromMilliseconds(20)).WithNumberOfBytesToDelaySending(1).Build(); + } + + return null; + } + } - public string NetworkConditionDescription { get; } + public string NetworkConditionDescription { get; private set; } - public string ShortNetworkConditionDescription { get; } + public string ShortNetworkConditionDescription { get; private set; } + + public NetworkConditionTestCase() + { + + } - public NetworkConditionTestCase(Func? portForwarderFactory, string networkConditionDescription, string shortNetworkConditionDescription) + public NetworkConditionTestCase(int factory, string networkConditionDescription, string shortNetworkConditionDescription) { - PortForwarderFactory = portForwarderFactory; + this.factory = factory; NetworkConditionDescription = networkConditionDescription; ShortNetworkConditionDescription = shortNetworkConditionDescription; @@ -61,5 +83,19 @@ public string ToShortString() { return $"Net: '{ShortNetworkConditionDescription}'"; } + + public void Deserialize(IXunitSerializationInfo info) + { + factory = info.GetValue(nameof(factory)); + NetworkConditionDescription = info.GetValue(nameof(NetworkConditionDescription)); + ShortNetworkConditionDescription = info.GetValue(nameof(ShortNetworkConditionDescription)); + } + + public void Serialize(IXunitSerializationInfo info) + { + info.AddValue(nameof(factory), factory); + info.AddValue(nameof(NetworkConditionDescription), NetworkConditionDescription); + info.AddValue(nameof(ShortNetworkConditionDescription), ShortNetworkConditionDescription); + } } } \ No newline at end of file diff --git a/source/Halibut.Tests/Util/DoNotRunInParallelGlobally.cs b/source/Halibut.Tests/Util/DoNotRunInParallelGlobally.cs new file mode 100644 index 000000000..72c76f047 --- /dev/null +++ b/source/Halibut.Tests/Util/DoNotRunInParallelGlobally.cs @@ -0,0 +1,11 @@ +using System; +using Xunit; + +namespace Halibut.Tests.Util +{ + // CollectionDefinition does NOT WORK when put on a test class itself. So we make this separate collection to use CollectionDefinition. + [CollectionDefinition(nameof(DoNotRunInParallelGlobally), DisableParallelization = true)] + public class DoNotRunInParallelGlobally + { + } +} \ No newline at end of file diff --git a/source/Halibut.Tests/Util/ParallelTestFramework.cs b/source/Halibut.Tests/Util/ParallelTestFramework.cs new file mode 100644 index 000000000..14d1239c6 --- /dev/null +++ b/source/Halibut.Tests/Util/ParallelTestFramework.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Halibut.Tests.Util +{ + // To make this work, we have to hook it up as the TestFramework in AssemblyInfo (including namespace) + sealed class ParallelTestFramework : XunitTestFramework + { + public ParallelTestFramework(IMessageSink diagnosticMessageSink) + : base(diagnosticMessageSink) + { + } + + protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + { + return new CustomTestExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink); + } + + sealed class CustomTestExecutor : XunitTestFrameworkExecutor + { + public CustomTestExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink) + : base(assemblyName, sourceInformationProvider, diagnosticMessageSink) + { + } + + protected override async void RunTestCases(IEnumerable testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + { + var newTestCases = SetUpTestCaseParallelization(testCases); + + using var assemblyRunner = new XunitTestAssemblyRunner(TestAssembly, newTestCases, DiagnosticMessageSink, executionMessageSink, executionOptions); + await assemblyRunner.RunAsync(); + } + + /// + /// By default, all test cases in a test class share the same collection instance which ensures they run synchronously. + /// By providing a unique test collection instance to every test case in a test class you can make them all run in parallel. + /// + IEnumerable SetUpTestCaseParallelization(IEnumerable testCases) + { + var result = new List(); + foreach (var testCase in testCases) + { + var oldTestMethod = testCase.TestMethod; + var oldTestClass = oldTestMethod.TestClass; + var oldTestCollection = oldTestMethod.TestClass.TestCollection; + + // If the collection is explicitly set, don't try to parallelize test execution + if (oldTestCollection.CollectionDefinition != null || oldTestClass.Class.GetCustomAttributes(typeof(CollectionAttribute)).Any()) + { + result.Add(testCase); + continue; + } + + // Create a new collection with a unique id for the test case. + var newTestCollection = + new TestCollection( + oldTestCollection.TestAssembly, + oldTestCollection.CollectionDefinition, + displayName: $"{oldTestCollection.DisplayName} {oldTestCollection.UniqueID}"); + newTestCollection.UniqueID = Guid.NewGuid(); + + // Duplicate the test and assign it to the new collection + var newTestClass = new TestClass(newTestCollection, oldTestClass.Class); + var newTestMethod = new TestMethod(newTestClass, oldTestMethod.Method); + switch (testCase) + { + // Used by Theory having DisableDiscoveryEnumeration or non-serializable data + case XunitTheoryTestCase xunitTheoryTestCase: + result.Add(new XunitTheoryTestCase( + DiagnosticMessageSink, + GetTestMethodDisplay(xunitTheoryTestCase), + GetTestMethodDisplayOptions(xunitTheoryTestCase), + newTestMethod)); + break; + + // Used by all other tests + case XunitTestCase xunitTestCase: + result.Add(new XunitTestCase( + DiagnosticMessageSink, + GetTestMethodDisplay(xunitTestCase), + GetTestMethodDisplayOptions(xunitTestCase), + newTestMethod, + xunitTestCase.TestMethodArguments)); + break; + + default: + throw new ArgumentOutOfRangeException("Test case " + testCase.GetType() + " not supported"); + } + } + + return result; + + static TestMethodDisplay GetTestMethodDisplay(TestMethodTestCase testCase) + { + return (TestMethodDisplay)typeof(TestMethodTestCase) + .GetProperty("DefaultMethodDisplay", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(testCase)!; + } + + static TestMethodDisplayOptions GetTestMethodDisplayOptions(TestMethodTestCase testCase) + { + return (TestMethodDisplayOptions)typeof(TestMethodTestCase) + .GetProperty("DefaultMethodDisplayOptions", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(testCase)!; + } + } + } + } +} \ No newline at end of file