Respect TEST_RANDOM_SEED when --rng-seed is absent; add SelfTest & approvals

src/catch2/catch_config.hpp

@@ -61,6 +61,7 @@ namespace Catch {
 
         int abortAfter = -1;
         uint32_t rngSeed = generateRandomSeed(GenerateFrom::Default);
+        bool rngSeedSpecified = false;
 
         unsigned int shardCount = 1;
         unsigned int shardIndex = 0;

src/catch2/catch_session.cpp

@@ -29,6 +29,9 @@
 #include <exception>
 #include <iomanip>
 #include <set>
+#include <cstdlib>
+#include <cstdint>
+#include <climits>
 
 namespace Catch {
 
@@ -296,6 +299,17 @@ namespace Catch {
         }
 
         CATCH_TRY {
+        if (!m_configData.rngSeedSpecified) {
+    if (const char* v = std::getenv("TEST_RANDOM_SEED")) {
+        char* end = nullptr;
+        unsigned long long parsed = std::strtoull(v, &end, 10);
+        if (end != v && *end == '\0' && parsed <= UINT32_MAX) {
+            m_configData.rngSeed = static_cast<std::uint32_t>(parsed);
+            // intentionally NOT setting rngSeedSpecified; CLI should still override if provided
+        }
+        // malformed/out-of-range -> ignore and keep existing default
+    }
+}
             config(); // Force config to be constructed
 
             seedRng( *m_config );

src/catch2/internal/catch_commandline.cpp

@@ -72,9 +72,11 @@ namespace Catch {
         auto const setRngSeed = [&]( std::string const& seed ) {
                 if( seed == "time" ) {
                     config.rngSeed = generateRandomSeed(GenerateFrom::Time);
+                    config.rngSeedSpecified = true;
                     return ParserResult::ok(ParseResultType::Matched);
                 } else if (seed == "random-device") {
                     config.rngSeed = generateRandomSeed(GenerateFrom::RandomDevice);
+                    config.rngSeedSpecified = true;
                     return ParserResult::ok(ParseResultType::Matched);
                 }
 
@@ -85,6 +87,7 @@ namespace Catch {
                     return ParserResult::runtimeError( "Could not parse '" + seed + "' as seed" );
                 }
                 config.rngSeed = *parsedSeed;
+                config.rngSeedSpecified = true;
                 return ParserResult::ok( ParseResultType::Matched );
             };
         auto const setDefaultColourMode = [&]( std::string const& colourMode ) {

tests/CMakeLists.txt

@@ -69,6 +69,7 @@ endif() #Temporary workaround
 # Please keep these ordered alphabetically
 set(TEST_SOURCES
   ${SELF_TEST_DIR}/TestRegistrations.cpp
+  ${SELF_TEST_DIR}/BazelEnvSeed.tests.cpp 
   ${SELF_TEST_DIR}/IntrospectiveTests/Algorithms.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/AssertionHandler.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/Clara.tests.cpp
@@ -370,6 +371,14 @@ set_tests_properties(VersionCheck PROPERTIES PASS_REGULAR_EXPRESSION "Catch2 v${
 add_test(NAME LibIdentityTest COMMAND $<TARGET_FILE:SelfTest> --libidentify)
 set_tests_properties(LibIdentityTest PROPERTIES PASS_REGULAR_EXPRESSION "description:    A Catch2 test executable")
 
+add_test(NAME BazelEnvSeedTest
+  COMMAND $<TARGET_FILE:SelfTest> "[bazel-seed]"
+)
+set_tests_properties(BazelEnvSeedTest PROPERTIES
+  ENVIRONMENT "TEST_RANDOM_SEED=12345"
+  COST 1
+)
+
 add_test(NAME FilenameAsTagsTest COMMAND $<TARGET_FILE:SelfTest> -\# --list-tags)
 set_tests_properties(FilenameAsTagsTest PROPERTIES PASS_REGULAR_EXPRESSION "\\[#Approx.tests\\]")
 

tests/SelfTest/Baselines/automake.sw.approved.txt

@@ -144,6 +144,7 @@ Nor would this
 :test-result: FAIL EndsWith string matcher
 :test-result: PASS Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM
 :test-result: PASS Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM
+:test-result: PASS Env TEST_RANDOM_SEED sets seed when CLI not provided
 :test-result: PASS Epsilon only applies to Approx's value
 :test-result: XFAIL Equality checks that should fail
 :test-result: PASS Equality checks that should succeed

tests/SelfTest/Baselines/automake.sw.multi.approved.txt

@@ -142,6 +142,7 @@
 :test-result: FAIL EndsWith string matcher
 :test-result: PASS Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM
 :test-result: PASS Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM
+:test-result: PASS Env TEST_RANDOM_SEED sets seed when CLI not provided
 :test-result: PASS Epsilon only applies to Approx's value
 :test-result: XFAIL Equality checks that should fail
 :test-result: PASS Equality checks that should succeed

tests/SelfTest/Baselines/compact.sw.approved.txt

@@ -580,6 +580,7 @@ EnumToString.tests.cpp:<line number>: passed: stringify( EnumClass3::Value4 ) ==
 EnumToString.tests.cpp:<line number>: passed: stringify( ec3 ) == "Value2" for: "Value2" == "Value2"
 EnumToString.tests.cpp:<line number>: passed: stringify( Bikeshed::Colours::Red ) == "Red" for: "Red" == "Red"
 EnumToString.tests.cpp:<line number>: passed: stringify( Bikeshed::Colours::Blue ) == "Blue" for: "Blue" == "Blue"
+BazelEnvSeed.tests.cpp:<line number>: passed: with 1 message: 'TEST_RANDOM_SEED not set; skipping env-specific check'
 Approx.tests.cpp:<line number>: passed: 101.01 != Approx(100).epsilon(0.01) for: 101.01000000000000512 != Approx( 100.0 )
 Condition.tests.cpp:<line number>: failed: data.int_seven == 6 for: 7 == 6
 Condition.tests.cpp:<line number>: failed: data.int_seven == 8 for: 7 == 8
@@ -2888,7 +2889,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
 InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
 Misc.tests.cpp:<line number>: passed:
 Misc.tests.cpp:<line number>: passed:
-test cases:  435 |  317 passed |  95 failed | 6 skipped | 17 failed as expected
-assertions: 2303 | 2105 passed | 157 failed | 41 failed as expected
+test cases:  436 |  318 passed |  95 failed | 6 skipped | 17 failed as expected
+assertions: 2304 | 2106 passed | 157 failed | 41 failed as expected
 
 

tests/SelfTest/Baselines/compact.sw.multi.approved.txt

@@ -578,6 +578,7 @@ EnumToString.tests.cpp:<line number>: passed: stringify( EnumClass3::Value4 ) ==
 EnumToString.tests.cpp:<line number>: passed: stringify( ec3 ) == "Value2" for: "Value2" == "Value2"
 EnumToString.tests.cpp:<line number>: passed: stringify( Bikeshed::Colours::Red ) == "Red" for: "Red" == "Red"
 EnumToString.tests.cpp:<line number>: passed: stringify( Bikeshed::Colours::Blue ) == "Blue" for: "Blue" == "Blue"
+BazelEnvSeed.tests.cpp:<line number>: passed: with 1 message: 'TEST_RANDOM_SEED not set; skipping env-specific check'
 Approx.tests.cpp:<line number>: passed: 101.01 != Approx(100).epsilon(0.01) for: 101.01000000000000512 != Approx( 100.0 )
 Condition.tests.cpp:<line number>: failed: data.int_seven == 6 for: 7 == 6
 Condition.tests.cpp:<line number>: failed: data.int_seven == 8 for: 7 == 8
@@ -2877,7 +2878,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
 InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
 Misc.tests.cpp:<line number>: passed:
 Misc.tests.cpp:<line number>: passed:
-test cases:  435 |  317 passed |  95 failed | 6 skipped | 17 failed as expected
-assertions: 2303 | 2105 passed | 157 failed | 41 failed as expected
+test cases:  436 |  318 passed |  95 failed | 6 skipped | 17 failed as expected
+assertions: 2304 | 2106 passed | 157 failed | 41 failed as expected
 
 

tests/SelfTest/Baselines/console.std.approved.txt

@@ -1719,6 +1719,6 @@ due to unexpected exception with message:
   Why would you throw a std::string?
 
 ===============================================================================
-test cases:  435 |  335 passed |  76 failed | 7 skipped | 17 failed as expected
-assertions: 2282 | 2105 passed | 136 failed | 41 failed as expected
+test cases:  436 |  336 passed |  76 failed | 7 skipped | 17 failed as expected
+assertions: 2283 | 2106 passed | 136 failed | 41 failed as expected
 

tests/SelfTest/Baselines/console.sw.approved.txt

@@ -4259,6 +4259,16 @@ EnumToString.tests.cpp:<line number>: PASSED:
 with expansion:
   "Blue" == "Blue"
 
+-------------------------------------------------------------------------------
+Env TEST_RANDOM_SEED sets seed when CLI not provided
+-------------------------------------------------------------------------------
+BazelEnvSeed.tests.cpp:<line number>
+...............................................................................
+
+BazelEnvSeed.tests.cpp:<line number>: PASSED:
+with message:
+  TEST_RANDOM_SEED not set; skipping env-specific check
+
 -------------------------------------------------------------------------------
 Epsilon only applies to Approx's value
 -------------------------------------------------------------------------------
@@ -19295,6 +19305,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  435 |  317 passed |  95 failed | 6 skipped | 17 failed as expected
-assertions: 2303 | 2105 passed | 157 failed | 41 failed as expected
+test cases:  436 |  318 passed |  95 failed | 6 skipped | 17 failed as expected
+assertions: 2304 | 2106 passed | 157 failed | 41 failed as expected
 

tests/SelfTest/Baselines/console.sw.multi.approved.txt

@@ -4257,6 +4257,16 @@ EnumToString.tests.cpp:<line number>: PASSED:
 with expansion:
   "Blue" == "Blue"
 
+-------------------------------------------------------------------------------
+Env TEST_RANDOM_SEED sets seed when CLI not provided
+-------------------------------------------------------------------------------
+BazelEnvSeed.tests.cpp:<line number>
+...............................................................................
+
+BazelEnvSeed.tests.cpp:<line number>: PASSED:
+with message:
+  TEST_RANDOM_SEED not set; skipping env-specific check
+
 -------------------------------------------------------------------------------
 Epsilon only applies to Approx's value
 -------------------------------------------------------------------------------
@@ -19284,6 +19294,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  435 |  317 passed |  95 failed | 6 skipped | 17 failed as expected
-assertions: 2303 | 2105 passed | 157 failed | 41 failed as expected
+test cases:  436 |  318 passed |  95 failed | 6 skipped | 17 failed as expected
+assertions: 2304 | 2106 passed | 157 failed | 41 failed as expected
 

tests/SelfTest/Baselines/junit.sw.approved.txt

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="140" skipped="12" tests="2315" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="140" skipped="12" tests="2316" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="random-seed" value="1"/>
       <property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -575,6 +575,7 @@ at Matchers.tests.cpp:<line number>
     </testcase>
     <testcase classname="<exe-name>.global" name="Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Env TEST_RANDOM_SEED sets seed when CLI not provided" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Epsilon only applies to Approx's value" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Equality checks that should fail" time="{duration}" status="run">
       <skipped message="TEST_CASE tagged with !mayfail"/>

tests/SelfTest/Baselines/junit.sw.multi.approved.txt

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuites>
-  <testsuite name="<exe-name>" errors="17" failures="140" skipped="12" tests="2315" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="140" skipped="12" tests="2316" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="random-seed" value="1"/>
       <property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -574,6 +574,7 @@ at Matchers.tests.cpp:<line number>
     </testcase>
     <testcase classname="<exe-name>.global" name="Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Env TEST_RANDOM_SEED sets seed when CLI not provided" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Epsilon only applies to Approx's value" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Equality checks that should fail" time="{duration}" status="run">
       <skipped message="TEST_CASE tagged with !mayfail"/>

tests/SelfTest/Baselines/sonarqube.sw.approved.txt

@@ -2,6 +2,9 @@
 <!-- filters='"*" ~[!nonportable] ~[!benchmark] ~[approvals]' rng-seed=1 -->
 <testExecutions version="1"loose text artifact
 >
+  <file path="tests/<exe-name>/BazelEnvSeed.tests.cpp">
+    <testCase name="Env TEST_RANDOM_SEED sets seed when CLI not provided" duration="{duration}"/>
+  </file>
   <file path="tests/<exe-name>/IntrospectiveTests/AssertionHandler.tests.cpp">
     <testCase name="Assertions can be nested - CHECK" duration="{duration}">
       <skipped message="REQUIRE(i > 10)">

tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt

@@ -1,6 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!-- filters='"*" ~[!nonportable] ~[!benchmark] ~[approvals]' rng-seed=1 -->
 <testExecutions version="1">
+  <file path="tests/<exe-name>/BazelEnvSeed.tests.cpp">
+    <testCase name="Env TEST_RANDOM_SEED sets seed when CLI not provided" duration="{duration}"/>
+  </file>
   <file path="tests/<exe-name>/IntrospectiveTests/AssertionHandler.tests.cpp">
     <testCase name="Assertions can be nested - CHECK" duration="{duration}">
       <skipped message="REQUIRE(i > 10)">

tests/SelfTest/Baselines/tap.sw.approved.txt

@@ -1052,6 +1052,8 @@ ok {test-number} - stringify( ec3 ) == "Value2" for: "Value2" == "Value2"
 ok {test-number} - stringify( Bikeshed::Colours::Red ) == "Red" for: "Red" == "Red"
 # Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM
 ok {test-number} - stringify( Bikeshed::Colours::Blue ) == "Blue" for: "Blue" == "Blue"
+# Env TEST_RANDOM_SEED sets seed when CLI not provided
+ok {test-number} - with 1 message: 'TEST_RANDOM_SEED not set; skipping env-specific check'
 # Epsilon only applies to Approx's value
 ok {test-number} - 101.01 != Approx(100).epsilon(0.01) for: 101.01000000000000512 != Approx( 100.0 )
 # Equality checks that should fail
@@ -4627,5 +4629,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
 ok {test-number} -
 # xmlentitycheck
 ok {test-number} -
-1..2315
+1..2316
 

tests/SelfTest/Baselines/tap.sw.multi.approved.txt

@@ -1050,6 +1050,8 @@ ok {test-number} - stringify( ec3 ) == "Value2" for: "Value2" == "Value2"
 ok {test-number} - stringify( Bikeshed::Colours::Red ) == "Red" for: "Red" == "Red"
 # Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM
 ok {test-number} - stringify( Bikeshed::Colours::Blue ) == "Blue" for: "Blue" == "Blue"
+# Env TEST_RANDOM_SEED sets seed when CLI not provided
+ok {test-number} - with 1 message: 'TEST_RANDOM_SEED not set; skipping env-specific check'
 # Epsilon only applies to Approx's value
 ok {test-number} - 101.01 != Approx(100).epsilon(0.01) for: 101.01000000000000512 != Approx( 100.0 )
 # Equality checks that should fail
@@ -4616,5 +4618,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
 ok {test-number} -
 # xmlentitycheck
 ok {test-number} -
-1..2315
+1..2316
 

tests/SelfTest/Baselines/teamcity.sw.approved.txt

@@ -337,6 +337,8 @@
 ##teamcity[testFinished name='Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM' duration="{duration}"]
 ##teamcity[testStarted name='Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM']
 ##teamcity[testFinished name='Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM' duration="{duration}"]
+##teamcity[testStarted name='Env TEST_RANDOM_SEED sets seed when CLI not provided']
+##teamcity[testFinished name='Env TEST_RANDOM_SEED sets seed when CLI not provided' duration="{duration}"]
 ##teamcity[testStarted name='Epsilon only applies to Approx|'s value']
 ##teamcity[testFinished name='Epsilon only applies to Approx|'s value' duration="{duration}"]
 ##teamcity[testStarted name='Equality checks that should fail']

tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt

@@ -337,6 +337,8 @@
 ##teamcity[testFinished name='Enums can quickly have stringification enabled using CATCH_REGISTER_ENUM' duration="{duration}"]
 ##teamcity[testStarted name='Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM']
 ##teamcity[testFinished name='Enums in namespaces can quickly have stringification enabled using CATCH_REGISTER_ENUM' duration="{duration}"]
+##teamcity[testStarted name='Env TEST_RANDOM_SEED sets seed when CLI not provided']
+##teamcity[testFinished name='Env TEST_RANDOM_SEED sets seed when CLI not provided' duration="{duration}"]
 ##teamcity[testStarted name='Epsilon only applies to Approx|'s value']
 ##teamcity[testFinished name='Epsilon only applies to Approx|'s value' duration="{duration}"]
 ##teamcity[testStarted name='Equality checks that should fail']

tests/SelfTest/Baselines/xml.sw.approved.txt

@@ -4742,6 +4742,9 @@ C
     </Expression>
     <OverallResult success="true" skips="0"/>
   </TestCase>
+  <TestCase name="Env TEST_RANDOM_SEED sets seed when CLI not provided" tags="[bazel-seed]" filename="tests/<exe-name>/BazelEnvSeed.tests.cpp" >
+    <OverallResult success="true" skips="0"/>
+  </TestCase>
   <TestCase name="Epsilon only applies to Approx's value" tags="[Approx]" filename="tests/<exe-name>/UsageTests/Approx.tests.cpp" >
     <Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Approx.tests.cpp" >
       <Original>
@@ -22324,6 +22327,6 @@ Approx( -1.95996398454005449 )
     </Section>
     <OverallResult success="true" skips="0"/>
   </TestCase>
-  <OverallResults successes="2105" failures="157" expectedFailures="41" skips="12"/>
-  <OverallResultsCases successes="317" failures="95" expectedFailures="17" skips="6"/>
+  <OverallResults successes="2106" failures="157" expectedFailures="41" skips="12"/>
+  <OverallResultsCases successes="318" failures="95" expectedFailures="17" skips="6"/>
 </Catch2TestRun>

tests/SelfTest/Baselines/xml.sw.multi.approved.txt

@@ -4742,6 +4742,9 @@ C
     </Expression>
     <OverallResult success="true" skips="0"/>
   </TestCase>
+  <TestCase name="Env TEST_RANDOM_SEED sets seed when CLI not provided" tags="[bazel-seed]" filename="tests/<exe-name>/BazelEnvSeed.tests.cpp" >
+    <OverallResult success="true" skips="0"/>
+  </TestCase>
   <TestCase name="Epsilon only applies to Approx's value" tags="[Approx]" filename="tests/<exe-name>/UsageTests/Approx.tests.cpp" >
     <Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Approx.tests.cpp" >
       <Original>
@@ -22323,6 +22326,6 @@ Approx( -1.95996398454005449 )
     </Section>
     <OverallResult success="true" skips="0"/>
   </TestCase>
-  <OverallResults successes="2105" failures="157" expectedFailures="41" skips="12"/>
-  <OverallResultsCases successes="317" failures="95" expectedFailures="17" skips="6"/>
+  <OverallResults successes="2106" failures="157" expectedFailures="41" skips="12"/>
+  <OverallResultsCases successes="318" failures="95" expectedFailures="17" skips="6"/>
 </Catch2TestRun>

tests/SelfTest/BazelEnvSeed.tests.cpp

@@ -0,0 +1,31 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/interfaces/catch_interfaces_config.hpp>
+#include <catch2/internal/catch_context.hpp>
+#include <cstdlib>
+#include <cstdint>
+
+TEST_CASE("Env TEST_RANDOM_SEED sets seed when CLI not provided", "[bazel-seed]") {
+    auto const* cfg = Catch::getCurrentContext().getConfig();
+
+    const char* v = std::getenv("TEST_RANDOM_SEED");
+    if (!v) {
+
+        SUCCEED("TEST_RANDOM_SEED not set; skipping env-specific check");
+        return;
+    }
+
+    char* end = nullptr;
+    unsigned long long parsed = std::strtoull(v, &end, 10);
+    REQUIRE(end != v);
+    REQUIRE(*end == '\0');
+
+    REQUIRE(cfg->rngSeed() == static_cast<std::uint32_t>(parsed));
+}