diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index b7b1315ebf..1075e68c12 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -61,6 +61,7 @@ namespace Catch { int abortAfter = -1; uint32_t rngSeed = generateRandomSeed(GenerateFrom::Default); + bool rngSeedSpecified = false; unsigned int shardCount = 1; unsigned int shardIndex = 0; diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp index 7542447b9f..81ff10634d 100644 --- a/src/catch2/catch_session.cpp +++ b/src/catch2/catch_session.cpp @@ -29,6 +29,9 @@ #include #include #include +#include +#include +#include namespace Catch { @@ -296,6 +299,17 @@ namespace Catch { } CATCH_TRY { + if (!m_configData.rngSeedSpecified) { + if (const char* v = std::getenv("TEST_RANDOM_SEED")) { + char* end = nullptr; + unsigned long long parsed = std::strtoull(v, &end, 10); + if (end != v && *end == '\0' && parsed <= UINT32_MAX) { + m_configData.rngSeed = static_cast(parsed); + // intentionally NOT setting rngSeedSpecified; CLI should still override if provided + } + // malformed/out-of-range -> ignore and keep existing default + } +} config(); // Force config to be constructed seedRng( *m_config ); diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp index 212f17745a..65c9486183 100644 --- a/src/catch2/internal/catch_commandline.cpp +++ b/src/catch2/internal/catch_commandline.cpp @@ -72,9 +72,11 @@ namespace Catch { auto const setRngSeed = [&]( std::string const& seed ) { if( seed == "time" ) { config.rngSeed = generateRandomSeed(GenerateFrom::Time); + config.rngSeedSpecified = true; return ParserResult::ok(ParseResultType::Matched); } else if (seed == "random-device") { config.rngSeed = generateRandomSeed(GenerateFrom::RandomDevice); + config.rngSeedSpecified = true; return ParserResult::ok(ParseResultType::Matched); } @@ -85,6 +87,7 @@ namespace Catch { return ParserResult::runtimeError( "Could not parse '" + seed + "' as seed" ); } config.rngSeed = *parsedSeed; + config.rngSeedSpecified = true; return ParserResult::ok( ParseResultType::Matched ); }; auto const setDefaultColourMode = [&]( std::string const& colourMode ) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8f8a3755ae..a74ceb9b0a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -69,6 +69,7 @@ endif() #Temporary workaround # Please keep these ordered alphabetically set(TEST_SOURCES ${SELF_TEST_DIR}/TestRegistrations.cpp + ${SELF_TEST_DIR}/BazelEnvSeed.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Algorithms.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/AssertionHandler.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Clara.tests.cpp @@ -370,6 +371,14 @@ set_tests_properties(VersionCheck PROPERTIES PASS_REGULAR_EXPRESSION "Catch2 v${ add_test(NAME LibIdentityTest COMMAND $ --libidentify) set_tests_properties(LibIdentityTest PROPERTIES PASS_REGULAR_EXPRESSION "description: A Catch2 test executable") +add_test(NAME BazelEnvSeedTest + COMMAND $ "[bazel-seed]" +) +set_tests_properties(BazelEnvSeedTest PROPERTIES + ENVIRONMENT "TEST_RANDOM_SEED=12345" + COST 1 +) + add_test(NAME FilenameAsTagsTest COMMAND $ -\# --list-tags) set_tests_properties(FilenameAsTagsTest PROPERTIES PASS_REGULAR_EXPRESSION "\\[#Approx.tests\\]") diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 4a6886d6c2..b761f44ce3 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -144,6 +144,7 @@ Nor would this :test-result: FAIL EndsWith string matcher :test-result: PASS Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM :test-result: PASS Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM +:test-result: PASS Env TEST_RANDOM_SEED sets seed when CLI not provided :test-result: PASS Epsilon only applies to Approx's value :test-result: XFAIL Equality checks that should fail :test-result: PASS Equality checks that should succeed diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index b0d30b87d9..9e078a0fec 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -142,6 +142,7 @@ :test-result: FAIL EndsWith string matcher :test-result: PASS Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM :test-result: PASS Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM +:test-result: PASS Env TEST_RANDOM_SEED sets seed when CLI not provided :test-result: PASS Epsilon only applies to Approx's value :test-result: XFAIL Equality checks that should fail :test-result: PASS Equality checks that should succeed diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index cacb898afb..9c82f1b3da 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -580,6 +580,7 @@ EnumToString.tests.cpp:: passed: stringify( EnumClass3::Value4 ) == EnumToString.tests.cpp:: passed: stringify( ec3 ) == "Value2" for: "Value2" == "Value2" EnumToString.tests.cpp:: passed: stringify( Bikeshed::Colours::Red ) == "Red" for: "Red" == "Red" EnumToString.tests.cpp:: passed: stringify( Bikeshed::Colours::Blue ) == "Blue" for: "Blue" == "Blue" +BazelEnvSeed.tests.cpp:: passed: with 1 message: 'TEST_RANDOM_SEED not set; skipping env-specific check' Approx.tests.cpp:: passed: 101.01 != Approx(100).epsilon(0.01) for: 101.01000000000000512 != Approx( 100.0 ) Condition.tests.cpp:: failed: data.int_seven == 6 for: 7 == 6 Condition.tests.cpp:: failed: data.int_seven == 8 for: 7 == 8 @@ -2888,7 +2889,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 435 | 317 passed | 95 failed | 6 skipped | 17 failed as expected -assertions: 2303 | 2105 passed | 157 failed | 41 failed as expected +test cases: 436 | 318 passed | 95 failed | 6 skipped | 17 failed as expected +assertions: 2304 | 2106 passed | 157 failed | 41 failed as expected diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 10395c9adf..f94601aed0 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -578,6 +578,7 @@ EnumToString.tests.cpp:: passed: stringify( EnumClass3::Value4 ) == EnumToString.tests.cpp:: passed: stringify( ec3 ) == "Value2" for: "Value2" == "Value2" EnumToString.tests.cpp:: passed: stringify( Bikeshed::Colours::Red ) == "Red" for: "Red" == "Red" EnumToString.tests.cpp:: passed: stringify( Bikeshed::Colours::Blue ) == "Blue" for: "Blue" == "Blue" +BazelEnvSeed.tests.cpp:: passed: with 1 message: 'TEST_RANDOM_SEED not set; skipping env-specific check' Approx.tests.cpp:: passed: 101.01 != Approx(100).epsilon(0.01) for: 101.01000000000000512 != Approx( 100.0 ) Condition.tests.cpp:: failed: data.int_seven == 6 for: 7 == 6 Condition.tests.cpp:: failed: data.int_seven == 8 for: 7 == 8 @@ -2877,7 +2878,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 435 | 317 passed | 95 failed | 6 skipped | 17 failed as expected -assertions: 2303 | 2105 passed | 157 failed | 41 failed as expected +test cases: 436 | 318 passed | 95 failed | 6 skipped | 17 failed as expected +assertions: 2304 | 2106 passed | 157 failed | 41 failed as expected diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index c1512db5c9..326fdaf3f9 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1719,6 +1719,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 435 | 335 passed | 76 failed | 7 skipped | 17 failed as expected -assertions: 2282 | 2105 passed | 136 failed | 41 failed as expected +test cases: 436 | 336 passed | 76 failed | 7 skipped | 17 failed as expected +assertions: 2283 | 2106 passed | 136 failed | 41 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 7fd5951481..f95f198101 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -4259,6 +4259,16 @@ EnumToString.tests.cpp:: PASSED: with expansion: "Blue" == "Blue" +------------------------------------------------------------------------------- +Env TEST_RANDOM_SEED sets seed when CLI not provided +------------------------------------------------------------------------------- +BazelEnvSeed.tests.cpp: +............................................................................... + +BazelEnvSeed.tests.cpp:: PASSED: +with message: + TEST_RANDOM_SEED not set; skipping env-specific check + ------------------------------------------------------------------------------- Epsilon only applies to Approx's value ------------------------------------------------------------------------------- @@ -19295,6 +19305,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 435 | 317 passed | 95 failed | 6 skipped | 17 failed as expected -assertions: 2303 | 2105 passed | 157 failed | 41 failed as expected +test cases: 436 | 318 passed | 95 failed | 6 skipped | 17 failed as expected +assertions: 2304 | 2106 passed | 157 failed | 41 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 08ab1b2bef..6109e5c700 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -4257,6 +4257,16 @@ EnumToString.tests.cpp:: PASSED: with expansion: "Blue" == "Blue" +------------------------------------------------------------------------------- +Env TEST_RANDOM_SEED sets seed when CLI not provided +------------------------------------------------------------------------------- +BazelEnvSeed.tests.cpp: +............................................................................... + +BazelEnvSeed.tests.cpp:: PASSED: +with message: + TEST_RANDOM_SEED not set; skipping env-specific check + ------------------------------------------------------------------------------- Epsilon only applies to Approx's value ------------------------------------------------------------------------------- @@ -19284,6 +19294,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 435 | 317 passed | 95 failed | 6 skipped | 17 failed as expected -assertions: 2303 | 2105 passed | 157 failed | 41 failed as expected +test cases: 436 | 318 passed | 95 failed | 6 skipped | 17 failed as expected +assertions: 2304 | 2106 passed | 157 failed | 41 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index c12b61787b..e23e8b47db 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -575,6 +575,7 @@ at Matchers.tests.cpp: + diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index e844ca6cb0..b0663e2b5f 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -574,6 +574,7 @@ at Matchers.tests.cpp: + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index d79abd3748..1b7e279d1d 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -2,6 +2,9 @@ + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 98a06c1491..892e731d4d 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1,6 +1,9 @@ + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 3e5d18321a..7b146f5229 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -1052,6 +1052,8 @@ ok {test-number} - stringify( ec3 ) == "Value2" for: "Value2" == "Value2" ok {test-number} - stringify( Bikeshed::Colours::Red ) == "Red" for: "Red" == "Red" # Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM ok {test-number} - stringify( Bikeshed::Colours::Blue ) == "Blue" for: "Blue" == "Blue" +# Env TEST_RANDOM_SEED sets seed when CLI not provided +ok {test-number} - with 1 message: 'TEST_RANDOM_SEED not set; skipping env-specific check' # Epsilon only applies to Approx's value ok {test-number} - 101.01 != Approx(100).epsilon(0.01) for: 101.01000000000000512 != Approx( 100.0 ) # Equality checks that should fail @@ -4627,5 +4629,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2315 +1..2316 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 0fdb266a2a..a43c4e98a9 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -1050,6 +1050,8 @@ ok {test-number} - stringify( ec3 ) == "Value2" for: "Value2" == "Value2" ok {test-number} - stringify( Bikeshed::Colours::Red ) == "Red" for: "Red" == "Red" # Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM ok {test-number} - stringify( Bikeshed::Colours::Blue ) == "Blue" for: "Blue" == "Blue" +# Env TEST_RANDOM_SEED sets seed when CLI not provided +ok {test-number} - with 1 message: 'TEST_RANDOM_SEED not set; skipping env-specific check' # Epsilon only applies to Approx's value ok {test-number} - 101.01 != Approx(100).epsilon(0.01) for: 101.01000000000000512 != Approx( 100.0 ) # Equality checks that should fail @@ -4616,5 +4618,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2315 +1..2316 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index 853f3d8048..a609d54755 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -337,6 +337,8 @@ ##teamcity[testFinished name='Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM' duration="{duration}"] ##teamcity[testStarted name='Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM'] ##teamcity[testFinished name='Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM' duration="{duration}"] +##teamcity[testStarted name='Env TEST_RANDOM_SEED sets seed when CLI not provided'] +##teamcity[testFinished name='Env TEST_RANDOM_SEED sets seed when CLI not provided' duration="{duration}"] ##teamcity[testStarted name='Epsilon only applies to Approx|'s value'] ##teamcity[testFinished name='Epsilon only applies to Approx|'s value' duration="{duration}"] ##teamcity[testStarted name='Equality checks that should fail'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index 10e318de5c..efd8648393 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -337,6 +337,8 @@ ##teamcity[testFinished name='Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM' duration="{duration}"] ##teamcity[testStarted name='Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM'] ##teamcity[testFinished name='Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM' duration="{duration}"] +##teamcity[testStarted name='Env TEST_RANDOM_SEED sets seed when CLI not provided'] +##teamcity[testFinished name='Env TEST_RANDOM_SEED sets seed when CLI not provided' duration="{duration}"] ##teamcity[testStarted name='Epsilon only applies to Approx|'s value'] ##teamcity[testFinished name='Epsilon only applies to Approx|'s value' duration="{duration}"] ##teamcity[testStarted name='Equality checks that should fail'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 142d662016..3095525d6a 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -4742,6 +4742,9 @@ C + + + @@ -22324,6 +22327,6 @@ Approx( -1.95996398454005449 ) - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index 62562332c9..13125ee5a9 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -4742,6 +4742,9 @@ C + + + @@ -22323,6 +22326,6 @@ Approx( -1.95996398454005449 ) - - + + diff --git a/tests/SelfTest/BazelEnvSeed.tests.cpp b/tests/SelfTest/BazelEnvSeed.tests.cpp new file mode 100644 index 0000000000..b6e358decb --- /dev/null +++ b/tests/SelfTest/BazelEnvSeed.tests.cpp @@ -0,0 +1,31 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include +#include +#include +#include +#include + +TEST_CASE("Env TEST_RANDOM_SEED sets seed when CLI not provided", "[bazel-seed]") { + auto const* cfg = Catch::getCurrentContext().getConfig(); + + const char* v = std::getenv("TEST_RANDOM_SEED"); + if (!v) { + + SUCCEED("TEST_RANDOM_SEED not set; skipping env-specific check"); + return; + } + + char* end = nullptr; + unsigned long long parsed = std::strtoull(v, &end, 10); + REQUIRE(end != v); + REQUIRE(*end == '\0'); + + REQUIRE(cfg->rngSeed() == static_cast(parsed)); +} diff --git a/tests/SelfTest/TestRegistrations.cpp b/tests/SelfTest/TestRegistrations.cpp index 1a7be39d44..53b1edbdfe 100644 --- a/tests/SelfTest/TestRegistrations.cpp +++ b/tests/SelfTest/TestRegistrations.cpp @@ -13,6 +13,7 @@ #include + // Some example tag aliases CATCH_REGISTER_TAG_ALIAS("[@nhf]", "[failing]~[.]") CATCH_REGISTER_TAG_ALIAS("[@tricky]", "[tricky]~[.]")