Add Release Test Run Profiles (#856)

* Add Release Test Run Profiles

Add two new test run profiles, `Run Tests (Release Mode)` and
`Debug Tests (Release Mode)`. These are identical to their Debug mode
counterparts, except they pass `-c release` during the build.

Adds tests to ensure that tests are being run in either debug or release
mode. These can be quite slow, so they've been tagged with @slow.

@slow tests are not run during CI runs. The plan is in the future to add
a nightly test suite that runs the full suite. This should keep CI fast
while allowing us to catch issues that may manifest in expensive tests.

* Consistently check if test kind is debug

* Rename isDebug to isDebugging

* Run fast test suite in CI

Author: plemarquand

assets/test/defaultPackage/Tests/PackageTests/PackageTests.swift

@@ -24,6 +24,20 @@ final class MixedXCTestSuite: XCTestCase {
   }
 }
 
+final class DebugReleaseTestSuite: XCTestCase {
+  func testRelease() throws {
+    #if DEBUG
+      XCTFail("Test was run in debug mode.")
+    #endif
+  }
+
+  func testDebug() throws {
+    #if RELEASE
+      XCTFail("Test was run in release mode.")
+    #endif
+  }
+}
+
 #if swift(>=6.0)
 import Testing
 
@@ -37,6 +51,18 @@ func parameterizedTest(_ arg: Int) {
   #expect(arg != 2)
 }
 
+@Test func testRelease() throws {
+  #if DEBUG
+    Issue.record("Test was run in debug mode.")
+  #endif
+}
+
+@Test func testDebug() throws {
+  #if RELEASE
+    Issue.record("Test was run in release mode.")
+  #endif
+}
+
 @Suite
 struct MixedSwiftTestingSuite {
   @Test

docker/docker-compose.yaml

@@ -14,6 +14,7 @@ services:
     image: vscode-swift:default
     environment:
       - CI=1
+      - FAST_TEST_RUN=1
     depends_on: [runtime-setup]
     volumes:
       - ~/.ssh:/root/.ssh

package.json

@@ -1162,8 +1162,8 @@
     "watch": "tsc --watch",
     "lint": "eslint ./ --ext ts && tsc --noEmit",
     "format": "prettier --check src test",
-    "pretest": "npm run compile && npm run lint",
-    "test": "find ./assets/test -type d -name '.build' -exec rm -rf {} + && find . -type d -name 'Package.resolved' -exec rm -rf {} + && tsc -p ./ && node ./out/test/runTest.js",
+    "pretest": "npm run compile && npm run lint && find ./assets/test -type d -name '.build' -exec rm -rf {} + && find . -type d -name 'Package.resolved' -exec rm -rf {} + && tsc -p ./",
+    "test": "node ./out/test/runTest.js",
     "coverage": "c8 --clean npm run test",
     "compile-tests": "find ./assets/test -type d -name '.build' -exec rm -rf {} + && npm run compile && npm run esbuild",
     "package": "vsce package",
@@ -1201,4 +1201,4 @@
     "vscode-languageclient": "^9.0.1",
     "xml2js": "^0.6.2"
   }
-}
+}

src/TestExplorer/TestKind.ts

@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the VS Code Swift open source project
+//
+// Copyright (c) 2021-2024 the VS Code Swift project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of VS Code Swift project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+/** Workspace Folder events */
+export enum TestKind {
+    // run tests serially
+    standard = "Run Tests",
+    // run tests in parallel
+    parallel = "Run Tests (Parallel)",
+    // run tests and extract test coverage
+    coverage = "Run With Test Coverage",
+    // run tests with the debugger
+    debug = "Debug Tests",
+    // run tests compiled in release mode
+    release = "Run Tests (Release Mode)",
+    // run tests compiled in release mode with debugger
+    debugRelease = "Debug Tests (Release Mode)",
+}
+
+export function isDebugging(testKind: TestKind): boolean {
+    return testKind === TestKind.debug || testKind === TestKind.debugRelease;
+}
+
+export function isRelease(testKind: TestKind): boolean {
+    return testKind === TestKind.release || testKind === TestKind.debugRelease;
+}

src/TestExplorer/TestRunner.ts

@@ -38,25 +38,7 @@ import { TestClass, runnableTag, upsertTestItem } from "./TestDiscovery";
 import { TestCoverage } from "../coverage/LcovResults";
 import { TestingDebugConfigurationFactory } from "../debugger/buildConfig";
 import { SwiftExecution } from "../tasks/SwiftExecution";
-
-/** Workspace Folder events */
-export enum TestKind {
-    // run tests serially
-    standard = "Standard",
-    // run tests in parallel
-    parallel = "Parallel",
-    // run tests and extract test coverage
-    coverage = "Coverage",
-    // run tests and extract test coverage
-    debug = "Debug",
-}
-
-export enum RunProfileName {
-    run = "Run Tests",
-    runParallel = "Run Tests (Parallel)",
-    coverage = "Test Coverage",
-    debug = "Debug Tests",
-}
+import { TestKind, isDebugging, isRelease } from "./TestKind";
 
 export enum TestLibrary {
     xctest = "XCTest",
@@ -248,7 +230,7 @@ export class TestRunner {
     ) {
         this.testArgs = new TestRunArguments(
             this.ensureRequestIncludesTests(this.request),
-            testKind === TestKind.debug
+            isDebugging(testKind)
         );
         this.testRun = new TestRunProxy(request, controller, this.testArgs, folderContext);
         this.xcTestOutputParser =
@@ -291,9 +273,9 @@ export class TestRunner {
         onCreateTestRun: vscode.EventEmitter<TestRunProxy>
     ): vscode.TestRunProfile[] {
         return [
-            // Add non-debug profile
+            // Add non-debug profiles
             controller.createRunProfile(
-                RunProfileName.run,
+                TestKind.standard,
                 vscode.TestRunProfileKind.Run,
                 async (request, token) => {
                     const runner = new TestRunner(
@@ -308,9 +290,8 @@ export class TestRunner {
                 true,
                 runnableTag
             ),
-            // Add non-debug profile
             controller.createRunProfile(
-                RunProfileName.runParallel,
+                TestKind.parallel,
                 vscode.TestRunProfileKind.Run,
                 async (request, token) => {
                     const runner = new TestRunner(
@@ -325,9 +306,25 @@ export class TestRunner {
                 false,
                 runnableTag
             ),
+            controller.createRunProfile(
+                TestKind.release,
+                vscode.TestRunProfileKind.Run,
+                async (request, token) => {
+                    const runner = new TestRunner(
+                        TestKind.release,
+                        request,
+                        folderContext,
+                        controller
+                    );
+                    onCreateTestRun.fire(runner.testRun);
+                    await runner.runHandler(token);
+                },
+                false,
+                runnableTag
+            ),
             // Add coverage profile
             controller.createRunProfile(
-                RunProfileName.coverage,
+                TestKind.coverage,
                 vscode.TestRunProfileKind.Coverage,
                 async (request, token) => {
                     const runner = new TestRunner(
@@ -350,7 +347,7 @@ export class TestRunner {
             ),
             // Add debug profile
             controller.createRunProfile(
-                RunProfileName.debug,
+                TestKind.debug,
                 vscode.TestRunProfileKind.Debug,
                 async (request, token) => {
                     const runner = new TestRunner(
@@ -365,6 +362,22 @@ export class TestRunner {
                 false,
                 runnableTag
             ),
+            controller.createRunProfile(
+                TestKind.debugRelease,
+                vscode.TestRunProfileKind.Debug,
+                async (request, token) => {
+                    const runner = new TestRunner(
+                        TestKind.debugRelease,
+                        request,
+                        folderContext,
+                        controller
+                    );
+                    onCreateTestRun.fire(runner.testRun);
+                    await runner.runHandler(token);
+                },
+                false,
+                runnableTag
+            ),
         ];
     }
 
@@ -377,7 +390,7 @@ export class TestRunner {
     async runHandler(token: vscode.CancellationToken) {
         const runState = new TestRunnerTestRunState(this.testRun);
         try {
-            if (this.testKind === TestKind.debug) {
+            if (isDebugging(this.testKind)) {
                 await this.debugSession(token, runState);
             } else {
                 await this.runSession(token, runState);
@@ -549,7 +562,13 @@ export class TestRunner {
                     kindLabel = " In Parallel";
                     break;
                 case TestKind.debug:
-                    kindLabel = "For Debugging";
+                    kindLabel = " For Debugging";
+                    break;
+                case TestKind.release:
+                    kindLabel = " in Release Mode";
+                    break;
+                case TestKind.debugRelease:
+                    kindLabel = " For Debugging in Release Mode";
                     break;
                 case TestKind.standard:
                     kindLabel = "";
@@ -662,6 +681,17 @@ export class TestRunner {
             return;
         }
 
+        if (isRelease(this.testKind)) {
+            buildAllTask.definition.args = [
+                ...buildAllTask.definition.args,
+                "-c",
+                "release",
+                "-Xswiftc",
+                "-enable-testing",
+            ];
+            buildAllTask.detail = `swift ${buildAllTask.definition.args.join(" ")}`;
+        }
+
         const subscriptions: vscode.Disposable[] = [];
         let buildExitCode = 0;
         const buildTask = vscode.tasks.onDidStartTask(e => {
@@ -699,20 +729,21 @@ export class TestRunner {
                 const swiftTestBuildConfig = TestingDebugConfigurationFactory.swiftTestingConfig(
                     this.folderContext,
                     fifoPipePath,
-                    TestKind.debug,
+                    this.testKind,
                     this.testArgs.swiftTestArgs,
                     true
                 );
 
                 if (swiftTestBuildConfig !== null) {
-                    // output test build configuration
-                    if (configuration.diagnostics) {
-                        const configJSON = JSON.stringify(swiftTestBuildConfig);
-                        this.workspaceContext.outputChannel.logDiagnostic(
-                            `swift-testing Debug Config: ${configJSON}`,
-                            this.folderContext.name
-                        );
-                    }
+                    swiftTestBuildConfig.testType = TestLibrary.swiftTesting;
+                    swiftTestBuildConfig.preLaunchTask = null;
+
+                    // If we're testing in both frameworks we're going to start more than one debugging
+                    // session. If both build configurations have the same name LLDB will replace the
+                    // output of the first one in the Debug Console with the output of the second one.
+                    // If they each have a unique name the Debug Console gets a nice dropdown the user
+                    // can switch between to see the output for both sessions.
+                    swiftTestBuildConfig.name = `Swift Testing: ${swiftTestBuildConfig.name}`;
 
                     // output test build configuration
                     if (configuration.diagnostics) {
@@ -723,15 +754,6 @@ export class TestRunner {
                         );
                     }
 
-                    swiftTestBuildConfig.testType = TestLibrary.swiftTesting;
-                    swiftTestBuildConfig.preLaunchTask = null;
-
-                    // If we're testing in both frameworks we're going to start more than one debugging
-                    // session. If both build configurations have the same name LLDB will replace the
-                    // output of the first one in the Debug Console with the output of the second one.
-                    // If they each have a unique name the Debug Console gets a nice dropdown the user
-                    // can switch between to see the output for both sessions.
-                    swiftTestBuildConfig.name = `Swift Testing: ${swiftTestBuildConfig.name}`;
                     buildConfigs.push(swiftTestBuildConfig);
                 }
             }
@@ -740,12 +762,16 @@ export class TestRunner {
             if (this.testArgs.hasXCTests) {
                 const xcTestBuildConfig = TestingDebugConfigurationFactory.xcTestConfig(
                     this.folderContext,
-                    TestKind.debug,
+                    this.testKind,
                     this.testArgs.xcTestArgs,
                     true
                 );
 
                 if (xcTestBuildConfig !== null) {
+                    xcTestBuildConfig.testType = TestLibrary.xctest;
+                    xcTestBuildConfig.preLaunchTask = null;
+                    xcTestBuildConfig.name = `XCTest: ${xcTestBuildConfig.name}`;
+
                     // output test build configuration
                     if (configuration.diagnostics) {
                         const configJSON = JSON.stringify(xcTestBuildConfig);
@@ -755,9 +781,6 @@ export class TestRunner {
                         );
                     }
 
-                    xcTestBuildConfig.testType = TestLibrary.xctest;
-                    xcTestBuildConfig.preLaunchTask = null;
-                    xcTestBuildConfig.name = `XCTest: ${xcTestBuildConfig.name}`;
                     buildConfigs.push(xcTestBuildConfig);
                 }
             }