OSOE-935: Upgrade to xUnit 3 and handling test cancellations gracefully

Lombiq.Tests.UI.Samples/FrontendUITestBase.cs

@@ -3,7 +3,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
-using Xunit.Abstractions;
+using Xunit;
 
 namespace Lombiq.Tests.UI.Samples;
 

Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj

@@ -3,6 +3,7 @@
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
+    <OutputType>Exe</OutputType>
   </PropertyGroup>
 
   <ItemGroup>
@@ -25,8 +26,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
+    <PackageReference Include="xunit.v3" Version="1.0.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="3.0.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

Lombiq.Tests.UI.Samples/Tests/AccessibilityTest.cs

@@ -2,7 +2,6 @@
 using Lombiq.Tests.UI.Services;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs

@@ -2,7 +2,6 @@
 using Lombiq.Tests.UI.Samples.Extensions;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/BasicOrchardFeaturesTests.cs

@@ -2,7 +2,6 @@
 using Lombiq.Tests.UI.Samples.Constants;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/BasicTests.cs

@@ -4,7 +4,6 @@
 using Shouldly;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests.cs

@@ -5,7 +5,6 @@
 using OpenQA.Selenium;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs

@@ -4,7 +4,6 @@
 using System.IO;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/EmailTests.cs

@@ -2,7 +2,6 @@
 using Lombiq.Tests.UI.Helpers;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs

@@ -1,6 +1,5 @@
 using Lombiq.Tests.UI.Exceptions;
 using Lombiq.Tests.UI.Extensions;
-using Lombiq.Tests.UI.Helpers;
 using Lombiq.Tests.UI.Pages;
 using Lombiq.Tests.UI.Samples.Helpers;
 using Lombiq.Tests.UI.Services;
@@ -10,7 +9,6 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 
@@ -41,7 +39,7 @@ public Task ServerSideErrorOnLoadedPageShouldHaltTest() =>
                 catch (PageChangeAssertionException)
                 {
                     // Remove all logs to have a clean slate.
-                    await context.ClearLogsAsync();
+                    await context.ClearLogsAsync(context.Configuration.TestCancellationToken);
                 }
             });
 
@@ -93,7 +91,7 @@ public Task BrowserLogsShouldPersist() =>
                 WriteConsoleLog();
 
                 // Since the browser log is updated asynchronously, we have to wait for most recent entries to appear.
-                ReliabilityHelper.DoWithRetriesOrFail(() =>
+                context.DoWithRetriesOrFail(() =>
                     context
                     .CumulativeBrowserLog
                     .Count(entry => entry.Text.Contains(testLog)) == 6);

Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs

@@ -2,7 +2,6 @@
 using OpenQA.Selenium;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs

@@ -1,10 +1,8 @@
 using Lombiq.Tests.UI.Extensions;
-using OpenQA.Selenium;
 using Shouldly;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 
@@ -33,40 +31,19 @@ public Task SampleTest() =>
                 // One use-case of interactive mode is to look around in the admin dashboard and troubleshoot the
                 // current settings without programmatically navigating there.
                 await context.SignInDirectlyAndGoToDashboardAsync();
-                await context.SwitchToInteractiveAsync();
+                await context.SwitchToInteractiveAsync(cancellationToken: context.Configuration.TestCancellationToken);
 
                 // Afterwards if you can still evaluate code as normal so `SignInDirectlyAndGoToDashboardAsync()` can be
                 // inserted anywhere. Bare in mind, that it's safest to use it before code that's already going to
                 // navigate away, like `GetCurrentUserNameAsync()` here. This ensures any manual navigation you did
                 // won't affect the test. If that's not practical, you can also do your thing in a new tab and close it
                 // before continuing the test.
                 (await context.GetCurrentUserNameAsync()).ShouldNotBeNullOrWhiteSpace();
-            });
 
-    // This test checks if interactive mode works by opening it in one thread, and then clicking it away in a different
-    // thread. Two threads are necessary because interactive mode stops test execution on its current thread, so we
-    // wouldn't be able to end it from within a test.
-    [Fact]
-    public Task EnteringInteractiveModeShouldWait() =>
-        ExecuteTestAfterSetupAsync(
-            async context =>
-            {
-                var currentUrl = context.Driver.Url;
-
-                await Task.WhenAll(
-                    context.SwitchToInteractiveAsync(),
-                    Task.Run(async () =>
-                    {
-                        // Ensure that the interactive mode polls for status at least once, so the arbitrary waiting
-                        // actually works in a real testing scenario.
-                        await Task.Delay(5000);
-
-                        await context.ClickReliablyOnAsync(By.ClassName("interactive__continue"));
-                    }));
-
-                // Ensure that the info tab is closed and the control handed back to the last tab.
-                context.Driver.Url.ShouldBe(currentUrl);
-            });
+                // Interactive mode only makes sense if you can see the browser, so below we make sure it's not running
+                // in headless mode.
+            },
+            configuration => configuration.BrowserConfiguration.Headless = false);
 }
 
 // END OF TRAINING SECTION: Interactive mode.

Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs

@@ -2,7 +2,6 @@
 using System.IO;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/MultiBrowserTests.cs

@@ -5,7 +5,6 @@
 using Shouldly;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/RemoteTests.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 
@@ -13,13 +12,26 @@ namespace Lombiq.Tests.UI.Samples.Tests;
 // to the internet.
 
 // However, sometimes in addition to this, you also want to test remote apps available online, like running rudimentary
-// smoke tests on your production app (e.g.: Can people still log in? Are payments still working?). The UI Testing
+// smoke tests on your production app (e.g.: Can people still log in? Are payments still working?). You can also run
+// such tests in your pre-production environment (like staging) before rolling it out the production. The UI Testing
 // Toolbox also supports this. Check out the example below!
 
 // Note how the test derives from RemoteUITestBase this time, not UITestBase. This is a generic base class for any
 // remote test. However, if you're testing an app behind Cloudflare, you might see requests rejected with an HTTP 403
 // due to Cloudflare's Bot Fight Mode; in that case, derive from CloudflareRemoteUITestBase instead after checking out
 // its documentation on how to set it up.
+
+// While there may be some common code between local and remote tests, they cannot be the same, and the common code must
+// be in a project independent of the web app (which is not the case here for the sake of simplicity):
+// - In local tests, you have a much lower-level access to the app: E.g., you can access the Orchard Core log, use
+//   shortcuts with all kinds of backdoors, and you can communicate directly with the Orchard Core shell (not just via #spell-check-ignore-line
+//   web APIs). You can also save a snapshot of the site's database and media. None of these are available (for a good
+//   reason) when the app is deployed to a server: There, you interact with it just as any ordinary user.
+// - For local tests, you need to build the web app and all its dependencies. For remote tests, you only need the test
+//   project and the UI Testing Toolbox.
+// - Some things are inherently different in a staging/production app, also on the UI. E.g., the app will load static
+//   resources from a CDN, or the content will be more up-to-date.
+
 public class RemoteTests : RemoteUITestBase
 {
     public RemoteTests(ITestOutputHelper testOutputHelper)

Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs

@@ -7,7 +7,6 @@
 using System;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 using YamlDotNet.RepresentationModel;
 
 namespace Lombiq.Tests.UI.Samples.Tests;

Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs

@@ -6,7 +6,6 @@
 using System.Globalization;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs

@@ -2,7 +2,6 @@
 using Lombiq.Tests.UI.Samples.Extensions;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/TenantTests.cs

@@ -5,7 +5,6 @@
 using Shouldly;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/UITestBase.cs

@@ -5,7 +5,7 @@
 using Microsoft.Extensions.Logging;
 using System;
 using System.Threading.Tasks;
-using Xunit.Abstractions;
+using Xunit;
 
 namespace Lombiq.Tests.UI.Samples;
 
@@ -114,11 +114,12 @@ protected override Task ExecuteTestAsync(
                 // JavaScript errors show up) are checked and if there are any errors, the test will fail. You can also
                 // enable the checking of accessibility rules as we'll see later. Maybe not all of the default checks
                 // are suitable for you. Then it's simple to override them; here we change which log entries cause the
-                // tests to fail, and allow certain log entries.
+                // tests to fail, and allow certain log entries: info-level log entries for ShellHost are allowed, see
+                // above.
                 configuration.AssertAppLogsAsync = webApplicationInstance =>
-                    webApplicationInstance.LogsShouldNotContainAsync(logEntry =>
-                    // Allowing info-level log entries for ShellHost, see above.
-                    logEntry.Message != "My permitted message." && logEntry.Level != LogLevel.Information);
+                    webApplicationInstance.LogsShouldNotContainAsync(
+                        logEntry => logEntry.Message != "My permitted message." && logEntry.Level != LogLevel.Information,
+                        configuration.TestCancellationToken);
 
                 // Strictly speaking this is not necessary here, because we always use the same static method for setup.
                 // However, if you used a dynamic setup operation (e.g. `context => SetupHelpers.RunSetupAsync(context,

Lombiq.Tests.UI.Tests.UI/TestCases/CustomAdminPrefixTestCases.cs

@@ -22,13 +22,21 @@ public static Task NavigationWithCustomAdminPrefixShouldWorkAsync(
             configuration =>
             {
                 configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
+
                 configuration.OrchardCoreConfiguration.BeforeAppStart += (_, argsBuilder) =>
                 {
                     argsBuilder.AddWithValue("OrchardCore:OrchardCore_Admin:AdminUrlPrefix", "custom-admin");
 
                     return Task.CompletedTask;
                 };
 
+                // Using a custom setup operation so the UI Testing Toolbox will consider it unique, and not reuse its
+                // snapshot with other tests if this one ends up running first. This is necessary because the
+                // AdminUrlPrefix is saved to the DB by the OrchardCore.AdminMenu feature, like for the admin menu's
+                // Blog item; we don't want the other tests to use that.
+                var originalSetupOperation = configuration.SetupConfiguration.SetupOperation;
+                configuration.SetupConfiguration.SetupOperation = context => originalSetupOperation(context);
+
                 return Task.CompletedTask;
             });
 }

Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs

@@ -48,7 +48,7 @@ public static Task AddUserToFakeRoleShouldThrowAsync(
                 await context.CreateUserAsync(UserUserName, DefaultUser.Password, UserEmail);
                 await context.AddUserToRoleAsync(UserUserName, FakeRole).ShouldThrowAsync<RoleNotFoundException>();
 
-                await context.ClearLogsAsync();
+                await context.ClearLogsAsync(context.Configuration.TestCancellationToken);
             },
             browser,
             ConfigurationHelper.DisableHtmlValidation);
@@ -61,7 +61,7 @@ public static Task AllowFakePermissionToRoleShouldThrowAsync(
                 await context.AddPermissionToRoleAsync(FakePermission, AuthorRole)
                     .ShouldThrowAsync<PermissionNotFoundException>();
 
-                await context.ClearLogsAsync();
+                await context.ClearLogsAsync(context.Configuration.TestCancellationToken);
             },
             browser,
             ConfigurationHelper.DisableHtmlValidation);

Lombiq.Tests.UI.Tests.UI/TestCases/TimeoutTestCases.cs

@@ -6,16 +6,17 @@ namespace Lombiq.Tests.UI.Tests.UI.TestCases;
 public static class TimeoutTestCases
 {
     public static Task TestRunTimeoutShouldThrowAsync(
-        ExecuteTestAfterSetupAsync executeTestAfterSetupAsync,
+        ExecuteTestAfterSetupWithoutBrowserAsync executeTestAfterSetupWithoutBrowserAsync,
         Browser browser = Browser.None) =>
         Should.ThrowAsync(
-            async () => await executeTestAfterSetupAsync(
-                context => Task.Delay(TimeSpan.FromSeconds(1)),
-                browser,
+            async () => await executeTestAfterSetupWithoutBrowserAsync(
+                context => Task.Delay(TimeSpan.FromSeconds(1), context.Configuration.TestCancellationToken),
                 configuration =>
                 {
                     configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
                     configuration.MaxRetryCount = 0;
+                    configuration.TestDumpConfiguration.CreateTestDump = false;
+                    configuration.SetupConfiguration.SetupOperation = context => Task.FromResult(new Uri("http://example.com"));
 
                     configuration.TimeoutConfiguration.TestRunTimeout = TimeSpan.FromMilliseconds(10);
 

Lombiq.Tests.UI/Attributes/AllBrowsersAttribute.cs

@@ -2,19 +2,28 @@
 using System;
 using System.Collections.Generic;
 using System.Reflection;
+using System.Threading.Tasks;
+using Xunit;
 using Xunit.Sdk;
+using Xunit.v3;
 
 namespace Lombiq.Tests.UI.Attributes;
 
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)]
 public sealed class AllBrowsersAttribute : DataAttribute
 {
-    public override IEnumerable<object[]> GetData(MethodInfo testMethod)
+    public override ValueTask<IReadOnlyCollection<ITheoryDataRow>> GetData(MethodInfo testMethod, DisposalTracker disposalTracker)
     {
         var browsers = (IEnumerable<Browser>)Enum.GetValues(typeof(Browser));
+        var dataRows = new List<ITheoryDataRow>();
+
         foreach (var browser in browsers)
         {
-            yield return new[] { browser as object };
+            dataRows.Add(new TheoryDataRow(browser));
         }
+
+        return new ValueTask<IReadOnlyCollection<ITheoryDataRow>>(dataRows.AsReadOnly());
     }
+
+    public override bool SupportsDiscoveryEnumeration() => true;
 }