Enhanced test fixtures for integration tests. (#1834)

- Part of #1833.
- Adds `ContainerFixture` with scoped resource lifecycle and cleanup in
place of implementation inheritance for test support functions. The
fixture also handles resource prefixing and uses a more ergonomic
`CommandResult` in place of a tuple for return values.
- `ImageWarmup` suite pre-pulls well-known images, and
`copyWarmupImage()` tags test-local refs, keeping the canonical image
store untouched.
- Three-phase `integration-new`: warmup, followed by concurrent tests
(managed by the swift test
`--experimental-maximum-parallelization-width` flag), followed by
serialized tests.
- `coverage-new` merges unit + integration-new profraw, replacing
`coverage` in CI as a migration progress indicator.
- Updates GH workflow so non-coverage invokes both the `integration` and
`integration-new` Makefile targets, while coverage runs invoke the
`coverage-new` target.

Author: jglogan

.github/workflows/common.yml

@@ -103,11 +103,11 @@ jobs:
 
       - name: Test the container project
         if: ${{ !inputs.coverage }}
-        run: make APP_ROOT="${APP_ROOT}" LOG_ROOT="${LOG_ROOT}" test install-kernel integration
+        run: make APP_ROOT="${APP_ROOT}" LOG_ROOT="${LOG_ROOT}" test install-kernel integration integration-new
 
       - name: Test the container project with coverage
         if: ${{ inputs.coverage }}
-        run: make APP_ROOT="${APP_ROOT}" LOG_ROOT="${LOG_ROOT}" install-kernel coverage
+        run: make APP_ROOT="${APP_ROOT}" LOG_ROOT="${LOG_ROOT}" install-kernel coverage-new
 
       - name: Extract coverage percentages
         if: ${{ inputs.coverage }}

Makefile

@@ -151,7 +151,7 @@ dsym:
 
 .PHONY: test
 test:
-	@$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --skip TestCLI
+	@$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --skip TestCLI --skip IntegrationTests
 
 .PHONY: install-kernel
 install-kernel:
@@ -194,6 +194,56 @@ define GENERATE_COV_REPORTS
 	@cat $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt
 endef
 
+# New integration test infrastructure.
+# PARALLEL_WIDTH controls --experimental-maximum-parallelization-width for the
+# concurrent pass. WARMUP_FILTER, CONCURRENT_FILTER, and GLOBAL_FILTER select
+# the three phases. Expand the filter lists as suites are migrated from CLITests.
+PARALLEL_WIDTH ?= 2
+WARMUP_FILTER = ImageWarmup
+CONCURRENT_FILTER = DemoConcurrentTests
+GLOBAL_FILTER = DemoGlobalTests
+
+INTEGRATION_SWIFT_EXTRA ?=
+INTEGRATION_POST_TEST ?=
+
+define RUN_INTEGRATION
+	@echo Ensuring apiserver stopped before the CLI integration tests...
+	@bin/container system stop && sleep 3 && scripts/ensure-container-stopped.sh
+	@if [ -n "$(APP_ROOT)" ]; then \
+		echo "Clearing application data under $(APP_ROOT) (preserving kernels)..." ; \
+		mkdir -p $(APP_ROOT) ; \
+		find "$(APP_ROOT)" -mindepth 1 -maxdepth 1 ! -name kernels -exec rm -rf {} + ; \
+	fi
+	@echo Running the integration tests...
+	@bin/container --debug system start --timeout 60 --enable-kernel-install $(SYSTEM_START_OPTS) && \
+	{ \
+		CLITEST_LOG_ROOT=$(LOG_ROOT) && export CLITEST_LOG_ROOT ; \
+		CONTAINER_CLI_PATH=$(ROOT_DIR)/bin/container && export CONTAINER_CLI_PATH ; \
+		echo "==> Warmup pass" && \
+		$(SWIFT) test $(INTEGRATION_SWIFT_EXTRA) -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter "$(WARMUP_FILTER)" && \
+		echo "==> Concurrent pass (width=$(PARALLEL_WIDTH))" && \
+		$(SWIFT) test $(INTEGRATION_SWIFT_EXTRA) -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --experimental-maximum-parallelization-width $(PARALLEL_WIDTH) --filter "$(CONCURRENT_FILTER)" && \
+		echo "==> Global pass (serial)" && \
+		$(SWIFT) test $(INTEGRATION_SWIFT_EXTRA) -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter "$(GLOBAL_FILTER)" ; \
+		exit_code=$$? ; \
+		$(INTEGRATION_POST_TEST) \
+		echo Ensuring apiserver stopped after the CLI integration tests ; \
+		scripts/ensure-container-stopped.sh ; \
+		exit $${exit_code} ; \
+	}
+endef
+
+.PHONY: integration-new
+integration-new: init-block
+	$(RUN_INTEGRATION)
+
+.PHONY: coverage-integration-new
+coverage-integration-new: INTEGRATION_SWIFT_EXTRA = --skip-build --enable-code-coverage
+coverage-integration-new: INTEGRATION_POST_TEST = cp $(COV_DATA_DIR)/*.profraw $(COVERAGE_OUTPUT_DIR)/integration/ ;
+coverage-integration-new: all
+	@mkdir -p $(COVERAGE_OUTPUT_DIR)/integration
+	$(RUN_INTEGRATION)
+
 INTEGRATION_TEST_SUITES ?= \
 	TestCLIHelp \
 	TestCLIStatus \
@@ -228,6 +278,21 @@ empty :=
 space := $(empty) $(empty)
 INTEGRATION_FILTER := $(subst $(space),|,$(strip $(INTEGRATION_TEST_SUITES)))
 
+.PHONY: coverage-new
+# Merges unit coverage with integration-new coverage. Use this during migration;
+# replace coverage with coverage-new in CI until all legacy tests are removed.
+coverage-new: coverage-build coverage-unit coverage-integration-new
+	@echo Merging integration coverage profdata...
+	@xcrun llvm-profdata merge -sparse $(COVERAGE_OUTPUT_DIR)/integration/*.profraw -o $(COVERAGE_OUTPUT_DIR)/integration/default.profdata
+	$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/integration/default.profdata,integration)
+	@echo Merging combined coverage profdata...
+	@mkdir -p $(COVERAGE_OUTPUT_DIR)/combined
+	@xcrun llvm-profdata merge -sparse \
+		$(COVERAGE_OUTPUT_DIR)/unit/default.profdata \
+		$(COVERAGE_OUTPUT_DIR)/integration/default.profdata \
+		-o $(COVERAGE_OUTPUT_DIR)/combined/default.profdata
+	$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/combined/default.profdata,combined)
+
 .PHONY: coverage-build
 coverage-build:
 	@echo Building tests with coverage instrumentation...
@@ -249,7 +314,7 @@ coverage-unit:
 	@echo Running unit test coverage...
 	@rm -f $(COV_DATA_DIR)/*.profraw
 	@mkdir -p $(COVERAGE_OUTPUT_DIR)/unit
-	@$(SWIFT) test --skip-build --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --skip TestCLI
+	@$(SWIFT) test --skip-build --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --skip TestCLI --skip IntegrationTests
 	@echo Merging unit coverage profdata...
 	@xcrun llvm-profdata merge -sparse $(COV_DATA_DIR)/*.profraw -o $(COVERAGE_OUTPUT_DIR)/unit/default.profdata
 	$(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/unit/default.profdata,unit)

Package.swift

@@ -80,6 +80,16 @@ let package = Package(
             ],
             path: "Sources/CLI"
         ),
+        .testTarget(
+            name: "IntegrationTests",
+            dependencies: [
+                .product(name: "Logging", package: "swift-log"),
+                .product(name: "SystemPackage", package: "swift-system"),
+                "ContainerLog",
+                "Yams",
+            ],
+            path: "Tests/IntegrationTests"
+        ),
         .testTarget(
             name: "CLITests",
             dependencies: [

Tests/IntegrationTests/Demo/DemoConcurrentTests.swift

@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2026 Apple Inc. and the container project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import Testing
+
+/// Demonstration suite for the concurrent test pass.
+///
+/// These eight tests run under ``--experimental-maximum-parallelization-width``
+/// to show bounded parallelism. Each test starts an isolated container (name
+/// scoped to its ``ContainerFixture/testID``) and sleeps for a random interval,
+/// so the total wall-clock time should be roughly max(individual durations)
+/// rather than their sum.
+///
+/// Delete this suite once real tests have been migrated to ``IntegrationTests``.
+@Suite
+struct DemoConcurrentTests {
+    @Test func test1() async throws { try await runDemo() }
+    @Test func test2() async throws { try await runDemo() }
+    @Test func test3() async throws { try await runDemo() }
+    @Test func test4() async throws { try await runDemo() }
+    @Test func test5() async throws { try await runDemo() }
+    @Test func test6() async throws { try await runDemo() }
+    @Test func test7() async throws { try await runDemo() }
+    @Test func test8() async throws { try await runDemo() }
+
+    private func runDemo() async throws {
+        try await ContainerFixture.with { f in
+            let image = try f.copyWarmupImage(ContainerFixture.warmupImages[0])
+            try await f.withContainer(image: image) { _ in
+                try await Task.sleep(for: .seconds(Int.random(in: 2...4)))
+            }
+        }
+    }
+}

Tests/IntegrationTests/Demo/DemoGlobalTests.swift

@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2026 Apple Inc. and the container project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import Testing
+
+/// Demonstration suite for the serial global test pass.
+///
+/// These two tests are structurally identical to ``DemoConcurrentTests`` but
+/// run under ``--experimental-maximum-parallelization-width 1`` in the Makefile
+/// to show serial execution. Total wall-clock time should be approximately the
+/// sum of the individual durations rather than the maximum.
+///
+/// Real global tests (image prune, system df, kernel set, etc.) will live here
+/// once migrated. Delete this suite at that point.
+@Suite
+struct DemoGlobalTests {
+    @Test func globalTest1() async throws { try await runDemo() }
+    @Test func globalTest2() async throws { try await runDemo() }
+
+    private func runDemo() async throws {
+        try await ContainerFixture.with { f in
+            let image = try f.copyWarmupImage(ContainerFixture.warmupImages[0])
+            try await f.withContainer(image: image) { _ in
+                try await Task.sleep(for: .seconds(Int.random(in: 2...4)))
+            }
+        }
+    }
+}

Tests/IntegrationTests/Utilities/CommandResult.swift

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2026 Apple Inc. and the container project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+struct CommandResult: Sendable {
+    let outputData: Data
+    let errorData: Data
+    let status: Int32
+
+    var output: String {
+        String(data: outputData, encoding: .utf8) ?? ""
+    }
+
+    var error: String {
+        String(data: errorData, encoding: .utf8) ?? ""
+    }
+
+    @discardableResult
+    func check(_ message: String? = nil) throws -> CommandResult {
+        guard status == 0 else {
+            let detail = message ?? error.trimmingCharacters(in: .whitespacesAndNewlines)
+            throw CommandError.nonZeroExit(status, detail)
+        }
+        return self
+    }
+}
+
+enum CommandError: Error {
+    case binaryNotFound
+    case executionFailed(String)
+    case nonZeroExit(Int32, String)
+}