diff --git a/ChangeLog.md b/ChangeLog.md index bdad312d..84035c07 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -5,6 +5,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Test shuffling support to detect hidden test dependencies (issue #530) + - `--shuffle` flag to randomize test execution order within each suite + - `--seed=N` option to specify random seed for reproducibility (0 uses time-based seed, implies --shuffle) + - Fisher-Yates shuffle algorithm implemented in `TestSuite` module + - Both unit tests and integration tests included + ## [4.16.0] - 2026-02-23 ### Added diff --git a/src/funit/FUnit.F90 b/src/funit/FUnit.F90 index ff896676..779d6249 100644 --- a/src/funit/FUnit.F90 +++ b/src/funit/FUnit.F90 @@ -54,6 +54,8 @@ logical function generic_run(load_tests, context) result(status) type(ArgParser), target :: parser logical :: debug logical :: xml + logical :: shuffle + integer :: random_seed type (StringUnlimitedMap) :: options class(*), pointer :: option integer :: unit @@ -133,6 +135,24 @@ logical function generic_run(load_tests, context) result(status) call apply_exclude_filters(option, suite, unit) end if + ! Apply shuffle if requested + shuffle = .false. + option => options%at('shuffle') + if (associated(option)) then + call cast(option, shuffle) + end if + + random_seed = 0 + option => options%at('random_seed') + if (associated(option)) then + call cast(option, random_seed) + if (random_seed /= 0) shuffle = .true. ! Seed implies shuffle + end if + + if (shuffle) then + call suite%set_shuffle(random_seed) + end if + r = runner%run(suite, context) status = r%wasSuccessful() @@ -376,8 +396,15 @@ subroutine set_command_line_options() & dest='tap_file', action='store', default=0, & & help='add a TAP listener and send results to file name') - call parser%add_argument('-x', '--xml', action='store_true', & - & help='print results with XmlPrinter') + call parser%add_argument('-x', '--xml', action='store_true', & + & help='print results with XmlPrinter') + + call parser%add_argument('--shuffle', action='store_true', & + & help='randomize test execution order within each suite') + + call parser%add_argument('--seed', type='integer', & + & dest='random_seed', action='store', default=0, & + & help='random seed for test shuffling (0=time-based, implies --shuffle)') #ifndef _GNU options = parser%parse_args() diff --git a/src/funit/core/TestSuite.F90 b/src/funit/core/TestSuite.F90 index 62d54546..5cd5fdf8 100644 --- a/src/funit/core/TestSuite.F90 +++ b/src/funit/core/TestSuite.F90 @@ -37,6 +37,8 @@ module PF_TestSuite !!$ private character(:), allocatable :: name type (TestVector) :: tests + logical :: shuffle_enabled = .false. + integer :: shuffle_seed = 0 contains procedure :: getName procedure :: setName @@ -49,6 +51,8 @@ module PF_TestSuite procedure :: getTestCases procedure :: filter procedure :: filter_sub + procedure :: set_shuffle + procedure :: shuffle_tests end type TestSuite interface TestSuite @@ -77,6 +81,8 @@ recursive subroutine copy(this, b) this%name = b%name this%tests = b%tests + this%shuffle_enabled = b%shuffle_enabled + this%shuffle_seed = b%shuffle_seed end subroutine copy @@ -104,6 +110,11 @@ recursive subroutine run(this, tstResult, context) class (Test), pointer :: t integer :: i + ! Shuffle tests if enabled + if (this%shuffle_enabled) then + call this%shuffle_tests() + end if + do i = 1, this%tests%size() t => this%tests%at(i) call t%run(tstResult, context) @@ -217,4 +228,74 @@ recursive function filter(this, a_filter) result(new_suite) end function filter + subroutine set_shuffle(this, seed) + class(TestSuite), intent(inout) :: this + integer, intent(in) :: seed + + this%shuffle_enabled = .true. + this%shuffle_seed = seed + end subroutine set_shuffle + + + subroutine shuffle_tests(this) + class(TestSuite), intent(inout) :: this + integer :: i, n + integer, allocatable :: indices(:) + type(TestVector) :: shuffled_tests + + n = this%tests%size() + if (n <= 1) return + + call initialize_random_seed(this%shuffle_seed) + + ! Create shuffled index array + indices = [(i, i=1, n)] + call shuffle_indices(indices) + + ! Build new vector in shuffled order + shuffled_tests = TestVector() + do i = 1, n + call shuffled_tests%push_back(this%tests%at(indices(i))) + end do + + this%tests = shuffled_tests + end subroutine shuffle_tests + + + subroutine initialize_random_seed(user_seed) + integer, intent(in) :: user_seed + integer, allocatable :: seed_array(:) + integer :: seed_size + + if (user_seed == 0) then + ! Use compiler's default random initialization + call random_seed() + return + end if + + ! Use user-provided seed + call random_seed(size=seed_size) + seed_array = spread(user_seed, 1, seed_size) + call random_seed(put=seed_array) + end subroutine initialize_random_seed + + + subroutine shuffle_indices(indices) + integer, intent(inout) :: indices(:) + integer :: i, j, n, temp + real :: rnd + + n = size(indices) + ! Fisher-Yates shuffle + do i = n, 2, -1 + call random_number(rnd) + j = int(rnd * i) + 1 + if (i == j) cycle + temp = indices(i) + indices(i) = indices(j) + indices(j) = temp + end do + end subroutine shuffle_indices + + end module PF_TestSuite diff --git a/tests/funit-core/CMakeLists.txt b/tests/funit-core/CMakeLists.txt index 9a9ccf89..a23cf996 100644 --- a/tests/funit-core/CMakeLists.txt +++ b/tests/funit-core/CMakeLists.txt @@ -68,6 +68,7 @@ set (test_srcs Test_TestResult.F90 Test_XmlPrinter.F90 Test_TestSuite.F90 + Test_TestSuite_Shuffle.F90 ) if (OPENMP_FORTRAN_FOUND) @@ -144,5 +145,21 @@ set_tests_properties(command_line_filtering PROPERTIES DEPENDS filter_cmdline_tests.x ) +# Shuffle integration tests +add_pfunit_ctest (shuffle_int_tests.x + TEST_SOURCES ShuffleIntegrationTests.pf + LINK_LIBRARIES other_shared +) +add_dependencies(build-tests shuffle_int_tests.x) + +add_test(NAME shuffle_integration + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test_shuffle.sh + ${CMAKE_CURRENT_BINARY_DIR}/shuffle_int_tests.x + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) +set_tests_properties(shuffle_integration PROPERTIES + DEPENDS shuffle_int_tests.x +) + diff --git a/tests/funit-core/ShuffleIntegrationTests.pf b/tests/funit-core/ShuffleIntegrationTests.pf new file mode 100644 index 00000000..1fbf2534 --- /dev/null +++ b/tests/funit-core/ShuffleIntegrationTests.pf @@ -0,0 +1,57 @@ +module ShuffleIntegrationTests + use FUnit + implicit none + +contains + + @test + subroutine test_01() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_02() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_03() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_04() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_05() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_06() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_07() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_08() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_09() + @assertTrue(.true.) + end subroutine + + @test + subroutine test_10() + @assertTrue(.true.) + end subroutine + +end module ShuffleIntegrationTests diff --git a/tests/funit-core/Test_TestSuite_Shuffle.F90 b/tests/funit-core/Test_TestSuite_Shuffle.F90 new file mode 100644 index 00000000..7e0e718d --- /dev/null +++ b/tests/funit-core/Test_TestSuite_Shuffle.F90 @@ -0,0 +1,277 @@ +#include "unused_dummy.fh" + +!------------------------------------------------------------------------------- +! NASA/GSFC, Advanced Software Technology Group +!------------------------------------------------------------------------------- +! MODULE: Test_TestSuite_Shuffle +! +!> @brief +!! Unit tests for TestSuite shuffle functionality +!! +!! @author +!! Tom Clune, NASA/GSFC +!! +!! @date +!! 10 Mar 2025 +! +!------------------------------------------------------------------------------- +module Test_TestSuite_Shuffle + use PF_TestSuite, only: TestSuite + use PF_TestResult + use PF_Assert, only: assertEqual, assertNotEqual, assertTrue, assertFalse + use PF_TestMethod, only: TestMethod + use PF_SerialContext + implicit none + private + + public :: suite + + ! Internal mock for TestResult that logs test execution order + type, extends(TestResult) :: LoggingResult + character(len=200) :: log + contains + procedure :: run => logging_run + end type LoggingResult + +contains + + function suite() + type (TestSuite) :: suite + + suite = TestSuite('TestSuite_Shuffle_Suite') + + call suite%addTest(TestMethod('test_no_shuffle_deterministic', & + & test_no_shuffle_deterministic)) + call suite%addTest(TestMethod('test_shuffle_seed_reproducible', & + & test_shuffle_seed_reproducible)) + call suite%addTest(TestMethod('test_shuffle_changes_order', & + & test_shuffle_changes_order)) + call suite%addTest(TestMethod('test_shuffle_empty_suite', & + & test_shuffle_empty_suite)) + call suite%addTest(TestMethod('test_shuffle_single_test', & + & test_shuffle_single_test)) + call suite%addTest(TestMethod('test_shuffle_preserves_suite_boundaries', & + & test_shuffle_preserves_suite_boundaries)) + end function suite + + + subroutine test_no_shuffle_deterministic() + ! Verify that without shuffle, tests run in deterministic order + type(TestSuite) :: test_suite + type(LoggingResult) :: result1, result2 + + test_suite = TestSuite('test') + call test_suite%addTest(TestMethod('t1', dummy_method)) + call test_suite%addTest(TestMethod('t2', dummy_method)) + call test_suite%addTest(TestMethod('t3', dummy_method)) + call test_suite%addTest(TestMethod('t4', dummy_method)) + call test_suite%addTest(TestMethod('t5', dummy_method)) + + result1%TestResult = TestResult() + result1%log = '' + result2%TestResult = TestResult() + result2%log = '' + + call test_suite%run(result1, SerialContext()) + call test_suite%run(result2, SerialContext()) + + ! Both runs should produce identical order + call assertEqual(result1%log, result2%log, 'Order should be deterministic without shuffle') + end subroutine test_no_shuffle_deterministic + + + subroutine test_shuffle_seed_reproducible() + ! Verify that same seed produces same order + type(TestSuite) :: suite1, suite2 + type(LoggingResult) :: result1, result2 + character(len=200) :: order1, order2 + integer :: i, pos1, pos2 + + suite1 = TestSuite('test1') + call suite1%addTest(TestMethod('t1', dummy_method)) + call suite1%addTest(TestMethod('t2', dummy_method)) + call suite1%addTest(TestMethod('t3', dummy_method)) + call suite1%addTest(TestMethod('t4', dummy_method)) + call suite1%addTest(TestMethod('t5', dummy_method)) + call suite1%addTest(TestMethod('t6', dummy_method)) + call suite1%addTest(TestMethod('t7', dummy_method)) + call suite1%addTest(TestMethod('t8', dummy_method)) + call suite1%addTest(TestMethod('t9', dummy_method)) + call suite1%addTest(TestMethod('t10', dummy_method)) + + suite2 = TestSuite('test1') ! Use same suite name for easier comparison + call suite2%addTest(TestMethod('t1', dummy_method)) + call suite2%addTest(TestMethod('t2', dummy_method)) + call suite2%addTest(TestMethod('t3', dummy_method)) + call suite2%addTest(TestMethod('t4', dummy_method)) + call suite2%addTest(TestMethod('t5', dummy_method)) + call suite2%addTest(TestMethod('t6', dummy_method)) + call suite2%addTest(TestMethod('t7', dummy_method)) + call suite2%addTest(TestMethod('t8', dummy_method)) + call suite2%addTest(TestMethod('t9', dummy_method)) + call suite2%addTest(TestMethod('t10', dummy_method)) + + ! Set same seed for both + call suite1%set_shuffle(12345) + call suite2%set_shuffle(12345) + + result1%TestResult = TestResult() + result1%log = '' + result2%TestResult = TestResult() + result2%log = '' + + call suite1%run(result1, SerialContext()) + call suite2%run(result2, SerialContext()) + + ! Same seed should produce same order + call assertEqual(result1%log, result2%log, 'Same seed should produce same order') + end subroutine test_shuffle_seed_reproducible + + + subroutine test_shuffle_changes_order() + ! Verify that shuffle actually changes order + ! With 10 tests, probability of same order is 1/10! ≈ 0.0000003% + type(TestSuite) :: test_suite + type(LoggingResult) :: no_shuffle_result, shuffle_result + logical :: order_changed + + test_suite = TestSuite('test') + call test_suite%addTest(TestMethod('t1', dummy_method)) + call test_suite%addTest(TestMethod('t2', dummy_method)) + call test_suite%addTest(TestMethod('t3', dummy_method)) + call test_suite%addTest(TestMethod('t4', dummy_method)) + call test_suite%addTest(TestMethod('t5', dummy_method)) + call test_suite%addTest(TestMethod('t6', dummy_method)) + call test_suite%addTest(TestMethod('t7', dummy_method)) + call test_suite%addTest(TestMethod('t8', dummy_method)) + call test_suite%addTest(TestMethod('t9', dummy_method)) + call test_suite%addTest(TestMethod('t10', dummy_method)) + + ! Run without shuffle first + no_shuffle_result%TestResult = TestResult() + no_shuffle_result%log = '' + call test_suite%run(no_shuffle_result, SerialContext()) + + ! Now enable shuffle with a fixed seed and run again + call test_suite%set_shuffle(99999) + shuffle_result%TestResult = TestResult() + shuffle_result%log = '' + call test_suite%run(shuffle_result, SerialContext()) + + ! Order should be different + order_changed = (no_shuffle_result%log /= shuffle_result%log) + call assertTrue(order_changed, 'Shuffle should change test order') + end subroutine test_shuffle_changes_order + + + subroutine test_shuffle_empty_suite() + ! Verify shuffle doesn't crash on empty suite + type(TestSuite) :: test_suite + type(LoggingResult) :: result + + test_suite = TestSuite('empty') + call test_suite%set_shuffle(12345) + + result%TestResult = TestResult() + result%log = '' + + ! Should not crash + call test_suite%run(result, SerialContext()) + + call assertEqual('', result%log, 'Empty suite should have empty log') + end subroutine test_shuffle_empty_suite + + + subroutine test_shuffle_single_test() + ! Verify shuffle works correctly with single test + type(TestSuite) :: test_suite + type(LoggingResult) :: result + + test_suite = TestSuite('single') + call test_suite%addTest(TestMethod('only_test', dummy_method)) + call test_suite%set_shuffle(12345) + + result%TestResult = TestResult() + result%log = '' + + call test_suite%run(result, SerialContext()) + + call assertTrue(index(result%log, 'only_test') > 0, 'Single test should run') + end subroutine test_shuffle_single_test + + + subroutine test_shuffle_preserves_suite_boundaries() + ! Verify that shuffle preserves suite boundaries (no inter-suite mixing) + type(TestSuite) :: parent_suite, childA, childB + type(LoggingResult) :: result + character(len=200) :: log_str + integer :: pos_a1, pos_a2, pos_b1, pos_b2 + + parent_suite = TestSuite('parent') + + childA = TestSuite('childA') + call childA%addTest(TestMethod('a1', dummy_method)) + call childA%addTest(TestMethod('a2', dummy_method)) + call childA%addTest(TestMethod('a3', dummy_method)) + + childB = TestSuite('childB') + call childB%addTest(TestMethod('b1', dummy_method)) + call childB%addTest(TestMethod('b2', dummy_method)) + call childB%addTest(TestMethod('b3', dummy_method)) + + call parent_suite%addTest(childA) + call parent_suite%addTest(childB) + + ! Enable shuffle on parent (note: current implementation only shuffles + ! tests within each suite, not the suites themselves) + call parent_suite%set_shuffle(54321) + + result%TestResult = TestResult() + result%log = '' + + call parent_suite%run(result, SerialContext()) + log_str = result%log + + ! All childA tests should appear before all childB tests + ! (or vice versa if suites are shuffled, but tests within suites shouldn't mix) + pos_a1 = index(log_str, 'childA.a1') + pos_a2 = index(log_str, 'childA.a2') + pos_b1 = index(log_str, 'childB.b1') + pos_b2 = index(log_str, 'childB.b2') + + ! Verify all tests were found + call assertTrue(pos_a1 > 0, 'childA.a1 should be in log') + call assertTrue(pos_a2 > 0, 'childA.a2 should be in log') + call assertTrue(pos_b1 > 0, 'childB.b1 should be in log') + call assertTrue(pos_b2 > 0, 'childB.b2 should be in log') + + ! Either all A's before all B's, or all B's before all A's + if (pos_a1 < pos_b1) then + ! A's should all come before B's + call assertTrue(pos_a2 < pos_b1, 'Suite boundaries should be preserved (A before B)') + else + ! B's should all come before A's + call assertTrue(pos_b2 < pos_a1, 'Suite boundaries should be preserved (B before A)') + end if + end subroutine test_shuffle_preserves_suite_boundaries + + + ! Helper methods + subroutine dummy_method() + ! Empty test method + end subroutine dummy_method + + + recursive subroutine logging_run(this, test, context) + use PF_TestCase + use PF_SurrogateTestCase + use PF_ParallelContext + class (LoggingResult), intent(inout) :: this + class (SurrogateTestCase), intent(inout) :: test + class (ParallelContext), intent(in) :: context + + _UNUSED_DUMMY(context) + this%log = trim(this%log)//' ::'//trim(test%getName()) + end subroutine logging_run + +end module Test_TestSuite_Shuffle diff --git a/tests/funit-core/serial_tests.F90 b/tests/funit-core/serial_tests.F90 index 2d40130d..d028c745 100644 --- a/tests/funit-core/serial_tests.F90 +++ b/tests/funit-core/serial_tests.F90 @@ -26,6 +26,7 @@ logical function runTests() result(success) use Test_TestResult, only: testResultSuite => suite ! (6) use Test_TestSuite, only: testTestSuiteSuite => suite ! (7) + use Test_TestSuite_Shuffle, only: testTestSuiteShuffleSuite => suite ! (7b) use Test_TestMethod, only: testTestMethodSuite => suite ! (8) use Test_SimpleTestCase, only: testSimpleTestCaseSuite => suite ! (9) @@ -56,6 +57,7 @@ logical function runTests() result(success) ADD(testResultSuite) ADD(testTestSuiteSuite) + ADD(testTestSuiteShuffleSuite) ADD(testTestMethodSuite) ADD(testSimpleTestCaseSuite) diff --git a/tests/funit-core/test_shuffle.sh b/tests/funit-core/test_shuffle.sh new file mode 100755 index 00000000..31025f9b --- /dev/null +++ b/tests/funit-core/test_shuffle.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Integration test for --shuffle and --seed command-line options + +set -e # Exit on error + +TEST_EXE="$1" +if [ -z "$TEST_EXE" ] || [ ! -x "$TEST_EXE" ]; then + echo "ERROR: Test executable not provided or not executable: $TEST_EXE" + exit 1 +fi + +# Color output helpers +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +FAILED=0 +PASSED=0 + +# Test function +run_test() { + local description="$1" + shift + + echo -n "Testing: $description ... " + + # Run test and capture output + if "$TEST_EXE" "$@" > /dev/null 2>&1; then + echo -e "${GREEN}PASS${NC}" + PASSED=$((PASSED + 1)) + return 0 + else + echo -e "${RED}FAIL${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi +} + +echo "==========================================" +echo "Testing pFUnit --shuffle and --seed Options" +echo "==========================================" +echo "" + +# Test 1: No shuffle - should run successfully +run_test "No shuffle (baseline)" + +# Test 2: With --shuffle - should run successfully +run_test "With --shuffle flag" --shuffle + +# Test 3: With --seed - should run successfully +run_test "With --seed=12345" --seed=12345 + +# Test 4: Verify --seed=N gives reproducible order +echo -n "Testing: Seed reproducibility ... " +output1=$("$TEST_EXE" --seed=54321 2>&1) +output2=$("$TEST_EXE" --seed=54321 2>&1) +if [ "$output1" = "$output2" ]; then + echo -e "${GREEN}PASS${NC}" + PASSED=$((PASSED + 1)) +else + echo -e "${RED}FAIL${NC} (outputs differ with same seed)" + FAILED=$((FAILED + 1)) +fi + +# Test 5: Verify multiple runs with different seeds all succeed +# Note: We can't easily verify order changes from output since pFUnit +# doesn't print test names in normal mode. The unit tests verify the +# actual shuffling behavior. +echo -n "Testing: Different seeds all run successfully ... " +success_count=0 +for seed in 111 222 333 444 555; do + if "$TEST_EXE" --seed=$seed > /dev/null 2>&1; then + success_count=$((success_count + 1)) + fi +done +if [ $success_count -eq 5 ]; then + echo -e "${GREEN}PASS${NC}" + PASSED=$((PASSED + 1)) +else + echo -e "${RED}FAIL${NC} (only $success_count/5 runs succeeded)" + FAILED=$((FAILED + 1)) +fi + +# Test 6: Verify --shuffle works multiple times +echo -n "Testing: Multiple --shuffle runs complete successfully ... " +success_count=0 +for i in {1..5}; do + if "$TEST_EXE" --shuffle > /dev/null 2>&1; then + success_count=$((success_count + 1)) + fi +done +if [ $success_count -eq 5 ]; then + echo -e "${GREEN}PASS${NC}" + PASSED=$((PASSED + 1)) +else + echo -e "${RED}FAIL${NC} (only $success_count/5 runs succeeded)" + FAILED=$((FAILED + 1)) +fi + +echo "" +echo "==========================================" +echo "Results: $PASSED passed, $FAILED failed" +echo "==========================================" + +if [ $FAILED -gt 0 ]; then + exit 1 +fi + +exit 0