From a41b23113c3c7fe6c36c777f71746435a27205d7 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Aug 2025 15:56:58 +0200 Subject: [PATCH 01/16] Make it possible for faulthandler to terminate the pytest process on timeout --- src/_pytest/faulthandler.py | 18 ++++++++++--- testing/test_faulthandler.py | 49 +++++++++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/_pytest/faulthandler.py b/src/_pytest/faulthandler.py index 79efc1d1704..f1138d78de8 100644 --- a/src/_pytest/faulthandler.py +++ b/src/_pytest/faulthandler.py @@ -16,11 +16,18 @@ def pytest_addoption(parser: Parser) -> None: - help = ( + help_timeout = ( "Dump the traceback of all threads if a test takes " "more than TIMEOUT seconds to finish" ) - parser.addini("faulthandler_timeout", help, default=0.0) + help_exit_on_timeout = ( + "Exit the test process if a test takes more than " + "faulthandler_timeout seconds to finish" + ) + parser.addini("faulthandler_timeout", help_timeout, default=0.0) + parser.addini( + "faulthandler_exit_on_timeout", help_exit_on_timeout, type="bool", default=False + ) def pytest_configure(config: Config) -> None: @@ -72,14 +79,19 @@ def get_timeout_config_value(config: Config) -> float: return float(config.getini("faulthandler_timeout") or 0.0) +def get_exit_on_timeout_config_value(config: Config) -> bool: + return bool(config.getini("faulthandler_exit_on_timeout")) + + @pytest.hookimpl(wrapper=True, trylast=True) def pytest_runtest_protocol(item: Item) -> Generator[None, object, object]: timeout = get_timeout_config_value(item.config) + exit_on_timeout = get_exit_on_timeout_config_value(item.config) if timeout > 0: import faulthandler stderr = item.config.stash[fault_handler_stderr_fd_key] - faulthandler.dump_traceback_later(timeout, file=stderr) + faulthandler.dump_traceback_later(timeout, file=stderr, exit=exit_on_timeout) try: return (yield) finally: diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index c416e81d2d9..4ece9793a6e 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -2,6 +2,7 @@ from __future__ import annotations import io +import os import sys from _pytest.pytester import Pytester @@ -71,20 +72,15 @@ def test_disabled(): assert result.ret == 0 -@pytest.mark.parametrize( - "enabled", - [ - pytest.param( - True, marks=pytest.mark.skip(reason="sometimes crashes on CI (#7022)") - ), - False, - ], -) +@pytest.mark.parametrize("enabled", [True, False]) def test_timeout(pytester: Pytester, enabled: bool) -> None: """Test option to dump tracebacks after a certain timeout. If faulthandler is disabled, no traceback will be dumped. """ + if enabled and "CI" in os.environ: + pytest.xfail(reason="sometimes crashes on CI (#7022)") + pytester.makepyfile( """ import os, time @@ -110,6 +106,41 @@ def test_timeout(): assert result.ret == 0 +@pytest.mark.parametrize("exit_on_timeout", [True, False]) +def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: + """Test option to force exit pytest process after a certain timeout.""" + if "CI" in os.environ: + pytest.xfail(reason="sometimes crashes on CI (#7022)") + + pytester.makepyfile( + """ + import os, time + def test_long_sleep_and_raise(): + time.sleep(1 if "CI" in os.environ else 0.1) + raise AssertionError( + "This test should have been interrupted before reaching this point." + ) + """ + ) + pytester.makeini( + f""" + [pytest] + faulthandler_timeout = 0.01 + faulthandler_exit_on_timeout = {"true" if exit_on_timeout else "false"} + """ + ) + result = pytester.runpytest_subprocess() + tb_output = "most recent call first" + result.stderr.fnmatch_lines([f"*{tb_output}*"]) + if exit_on_timeout: + result.stdout.no_fnmatch_line("*1 failed*") + result.stdout.no_fnmatch_line("*AssertionError*") + else: + result.stdout.fnmatch_lines(["*1 failed*"]) + result.stdout.fnmatch_lines(["*AssertionError*"]) + assert result.ret == 1 + + @pytest.mark.parametrize("hook_name", ["pytest_enter_pdb", "pytest_exception_interact"]) def test_cancel_timeout_on_hook(monkeypatch, hook_name) -> None: """Make sure that we are cancelling any scheduled traceback dumping due From 91d600cfe5617ae9dac558b6dd917aa2d3791c84 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Aug 2025 16:07:33 +0200 Subject: [PATCH 02/16] Add myself to AUTHORS and add changelog entry. --- AUTHORS | 1 + changelog/13678.feature.rst | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog/13678.feature.rst diff --git a/AUTHORS b/AUTHORS index d09c4dc98b4..73354bc765b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -338,6 +338,7 @@ Oleg Sushchenko Oleksandr Zavertniev Olga Matoula Oliver Bestwalter +Olivier Grisel Omar Kohl Omer Hadari Ondřej Súkup diff --git a/changelog/13678.feature.rst b/changelog/13678.feature.rst new file mode 100644 index 00000000000..7b34d9399f3 --- /dev/null +++ b/changelog/13678.feature.rst @@ -0,0 +1,3 @@ +Added a new `faulthandler_exit_on_timeout` ini option set to "false" by default to let `faulthandler` interrupt the `pytest` process after a timeout in case of deadlock. + +Previously, a `faulthandler` timeout would only dump the traceback of all threads to stderr, but would not interrupt the `pytest` process. From d3274b1ec1a44174ce1c7850f2cbef1e6bf73cfe Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Aug 2025 16:50:52 +0200 Subject: [PATCH 03/16] Update the documentation --- doc/en/reference/reference.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 7ec1b110baf..6f8e14995e6 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1484,6 +1484,25 @@ passed multiple times. The expected format is ``name=value``. For example:: For more information please refer to :ref:`faulthandler`. + +.. confval:: faulthandler_exit_on_timeout + + Exit the pytest process after the per-test timeout is reached by passing + `exit=True` to the :func:`faulthandler.dump_traceback_later` function. This + is particularly useful to avoid wasting CI resources for test suites that + are prone to putting the main Python interpreter into a deadlock state. + + This option is set to 'false' by default. + + .. code-block:: ini + + # content of pytest.ini + [pytest] + faulthandler_timeout=5 + faulthandler_exit_on_timeout=true + + For more information please refer to :ref:`faulthandler`. + .. confval:: filterwarnings @@ -2401,6 +2420,9 @@ All the command-line flags can be obtained by running ``pytest --help``:: faulthandler_timeout (string): Dump the traceback of all threads if a test takes more than TIMEOUT seconds to finish + faulthandler_exit_on_timeout (bool): + Exit the test process if a test takes more than + faulthandler_timeout seconds to finish addopts (args): Extra command line options minversion (string): Minimally required pytest version pythonpath (paths): Add paths to sys.path From be4f0bad8425d43d7e212e1514776edab0efc571 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Aug 2025 16:59:49 +0200 Subject: [PATCH 04/16] Skip flakky CI tests --- testing/test_faulthandler.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index 4ece9793a6e..81279997c83 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -2,7 +2,6 @@ from __future__ import annotations import io -import os import sys from _pytest.pytester import Pytester @@ -72,15 +71,20 @@ def test_disabled(): assert result.ret == 0 -@pytest.mark.parametrize("enabled", [True, False]) +@pytest.mark.parametrize( + "enabled", + [ + pytest.param( + True, marks=pytest.mark.skip(reason="sometimes crashes on CI (#7022)") + ), + False, + ], +) def test_timeout(pytester: Pytester, enabled: bool) -> None: """Test option to dump tracebacks after a certain timeout. If faulthandler is disabled, no traceback will be dumped. """ - if enabled and "CI" in os.environ: - pytest.xfail(reason="sometimes crashes on CI (#7022)") - pytester.makepyfile( """ import os, time @@ -106,12 +110,10 @@ def test_timeout(): assert result.ret == 0 +@pytest.mark.skip(reason="sometimes crashes on CI (#7022)") @pytest.mark.parametrize("exit_on_timeout", [True, False]) def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: """Test option to force exit pytest process after a certain timeout.""" - if "CI" in os.environ: - pytest.xfail(reason="sometimes crashes on CI (#7022)") - pytester.makepyfile( """ import os, time From 5066d988dbd37a7fbaf4e55463ae2ad8ce5294c4 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Aug 2025 17:49:16 +0200 Subject: [PATCH 05/16] Revert "Skip flakky CI tests" This reverts commit be4f0bad8425d43d7e212e1514776edab0efc571. --- testing/test_faulthandler.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index 81279997c83..4ece9793a6e 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -2,6 +2,7 @@ from __future__ import annotations import io +import os import sys from _pytest.pytester import Pytester @@ -71,20 +72,15 @@ def test_disabled(): assert result.ret == 0 -@pytest.mark.parametrize( - "enabled", - [ - pytest.param( - True, marks=pytest.mark.skip(reason="sometimes crashes on CI (#7022)") - ), - False, - ], -) +@pytest.mark.parametrize("enabled", [True, False]) def test_timeout(pytester: Pytester, enabled: bool) -> None: """Test option to dump tracebacks after a certain timeout. If faulthandler is disabled, no traceback will be dumped. """ + if enabled and "CI" in os.environ: + pytest.xfail(reason="sometimes crashes on CI (#7022)") + pytester.makepyfile( """ import os, time @@ -110,10 +106,12 @@ def test_timeout(): assert result.ret == 0 -@pytest.mark.skip(reason="sometimes crashes on CI (#7022)") @pytest.mark.parametrize("exit_on_timeout", [True, False]) def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: """Test option to force exit pytest process after a certain timeout.""" + if "CI" in os.environ: + pytest.xfail(reason="sometimes crashes on CI (#7022)") + pytester.makepyfile( """ import os, time From 000876ec177917d963e6c549728196a30b3f9f29 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Aug 2025 17:52:33 +0200 Subject: [PATCH 06/16] Pass the CI env in tox.ini and use it to conditionally skip tests on CI --- testing/test_faulthandler.py | 4 ++-- tox.ini | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index 4ece9793a6e..b0dc8e7ebfe 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -79,7 +79,7 @@ def test_timeout(pytester: Pytester, enabled: bool) -> None: If faulthandler is disabled, no traceback will be dumped. """ if enabled and "CI" in os.environ: - pytest.xfail(reason="sometimes crashes on CI (#7022)") + pytest.skip(reason="sometimes crashes on CI (#7022)") pytester.makepyfile( """ @@ -110,7 +110,7 @@ def test_timeout(): def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: """Test option to force exit pytest process after a certain timeout.""" if "CI" in os.environ: - pytest.xfail(reason="sometimes crashes on CI (#7022)") + pytest.skip(reason="sometimes crashes on CI (#7022)") pytester.makepyfile( """ diff --git a/tox.ini b/tox.ini index f1283aa8260..3fe7865a289 100644 --- a/tox.ini +++ b/tox.ini @@ -52,6 +52,7 @@ passenv = PYTEST_ADDOPTS TERM SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST + CI setenv = _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} {env:_PYTEST_FILES:} From dfc22076276471f45c33f3f6bfdaab62d2e43d77 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Aug 2025 18:09:33 +0200 Subject: [PATCH 07/16] Revert "Pass the CI env in tox.ini and use it to conditionally skip tests on CI" This reverts commit 000876ec177917d963e6c549728196a30b3f9f29. --- testing/test_faulthandler.py | 4 ++-- tox.ini | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index b0dc8e7ebfe..4ece9793a6e 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -79,7 +79,7 @@ def test_timeout(pytester: Pytester, enabled: bool) -> None: If faulthandler is disabled, no traceback will be dumped. """ if enabled and "CI" in os.environ: - pytest.skip(reason="sometimes crashes on CI (#7022)") + pytest.xfail(reason="sometimes crashes on CI (#7022)") pytester.makepyfile( """ @@ -110,7 +110,7 @@ def test_timeout(): def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: """Test option to force exit pytest process after a certain timeout.""" if "CI" in os.environ: - pytest.skip(reason="sometimes crashes on CI (#7022)") + pytest.xfail(reason="sometimes crashes on CI (#7022)") pytester.makepyfile( """ diff --git a/tox.ini b/tox.ini index 3fe7865a289..f1283aa8260 100644 --- a/tox.ini +++ b/tox.ini @@ -52,7 +52,6 @@ passenv = PYTEST_ADDOPTS TERM SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST - CI setenv = _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} {env:_PYTEST_FILES:} From c7abf41e8efaec7c31dae8366e4b77cdfda5eecc Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Aug 2025 18:09:43 +0200 Subject: [PATCH 08/16] Revert "Revert "Skip flakky CI tests"" This reverts commit 5066d988dbd37a7fbaf4e55463ae2ad8ce5294c4. --- testing/test_faulthandler.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index 4ece9793a6e..81279997c83 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -2,7 +2,6 @@ from __future__ import annotations import io -import os import sys from _pytest.pytester import Pytester @@ -72,15 +71,20 @@ def test_disabled(): assert result.ret == 0 -@pytest.mark.parametrize("enabled", [True, False]) +@pytest.mark.parametrize( + "enabled", + [ + pytest.param( + True, marks=pytest.mark.skip(reason="sometimes crashes on CI (#7022)") + ), + False, + ], +) def test_timeout(pytester: Pytester, enabled: bool) -> None: """Test option to dump tracebacks after a certain timeout. If faulthandler is disabled, no traceback will be dumped. """ - if enabled and "CI" in os.environ: - pytest.xfail(reason="sometimes crashes on CI (#7022)") - pytester.makepyfile( """ import os, time @@ -106,12 +110,10 @@ def test_timeout(): assert result.ret == 0 +@pytest.mark.skip(reason="sometimes crashes on CI (#7022)") @pytest.mark.parametrize("exit_on_timeout", [True, False]) def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: """Test option to force exit pytest process after a certain timeout.""" - if "CI" in os.environ: - pytest.xfail(reason="sometimes crashes on CI (#7022)") - pytester.makepyfile( """ import os, time From 713166ed60d10cb9b680cb34c2b30295ef3c1aa0 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 28 Aug 2025 10:16:44 +0200 Subject: [PATCH 09/16] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- changelog/13678.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/13678.feature.rst b/changelog/13678.feature.rst index 7b34d9399f3..63c7dfa399f 100644 --- a/changelog/13678.feature.rst +++ b/changelog/13678.feature.rst @@ -1,3 +1,3 @@ -Added a new `faulthandler_exit_on_timeout` ini option set to "false" by default to let `faulthandler` interrupt the `pytest` process after a timeout in case of deadlock. +Added a new `faulthandler_exit_on_timeout` ini option set to "false" by default to let `faulthandler` interrupt the `pytest` process after a timeout in case of deadlock -- by :user:`ogrisel`. Previously, a `faulthandler` timeout would only dump the traceback of all threads to stderr, but would not interrupt the `pytest` process. From da1adf323bacc43ae8e08916d616b2e4e65e0b73 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 28 Aug 2025 15:15:01 +0200 Subject: [PATCH 10/16] Better way to tell mypy that faulthandler_exit_on_timeout is always a bool instance --- src/_pytest/faulthandler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/_pytest/faulthandler.py b/src/_pytest/faulthandler.py index f1138d78de8..080cf583813 100644 --- a/src/_pytest/faulthandler.py +++ b/src/_pytest/faulthandler.py @@ -80,7 +80,9 @@ def get_timeout_config_value(config: Config) -> float: def get_exit_on_timeout_config_value(config: Config) -> bool: - return bool(config.getini("faulthandler_exit_on_timeout")) + exit_on_timeout = config.getini("faulthandler_exit_on_timeout") + assert isinstance(exit_on_timeout, bool) + return exit_on_timeout @pytest.hookimpl(wrapper=True, trylast=True) From 634ffcc7edd5faec9302b4638d2af38ce09f346b Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 3 Sep 2025 09:46:00 +0200 Subject: [PATCH 11/16] skip test only on CI --- testing/test_faulthandler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index df36e19d9b6..01dc260e78a 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -2,6 +2,7 @@ from __future__ import annotations import io +import os import sys from _pytest.pytester import Pytester @@ -76,7 +77,10 @@ def test_disabled(): "enabled", [ pytest.param( - True, marks=pytest.mark.skip(reason="sometimes crashes on CI (#7022)") + True, + marks=pytest.mark.skipif( + "CI" in os.environ, reason="sometimes crashes on CI (#7022)" + ), ), False, ], @@ -111,7 +115,8 @@ def test_timeout(): assert result.ret == 0 -@pytest.mark.skip(reason="sometimes crashes on CI (#7022)") +@pytest.mark.keep_ci_var +@pytest.mark.skipif("CI" in os.environ, reason="sometimes crashes on CI (#7022)") @pytest.mark.parametrize("exit_on_timeout", [True, False]) def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: """Test option to force exit pytest process after a certain timeout.""" From fbce0391fe44f666f244404616a6a1e8e8a57df1 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 9 Sep 2025 14:29:41 +0200 Subject: [PATCH 12/16] Trigger CI From ea997221cc289da77356b4d9b6bc69d1e72cf192 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 9 Sep 2025 14:43:27 +0200 Subject: [PATCH 13/16] Skip the new test on ubuntu-py314 From cfa2aa6eb21e8afe085da2b96bfc3967bdd08818 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 9 Sep 2025 14:57:06 +0200 Subject: [PATCH 14/16] Actually skip the new test on ubuntu-py314 --- testing/test_faulthandler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index 8f620755c4a..db9b092c568 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -119,6 +119,10 @@ def test_timeout(): @pytest.mark.keep_ci_var +@pytest.mark.skipif( + "CI" in os.environ and sys.platform == "linux" and sys.version_info >= (3, 14), + reason="sometimes crashes on CI because of truncated outputs (#7022)", +) @pytest.mark.parametrize("exit_on_timeout", [True, False]) def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: """Test option to force exit pytest process after a certain timeout.""" From 136b5150bd6993b159d8410012613185c4c24844 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 9 Sep 2025 15:11:45 +0200 Subject: [PATCH 15/16] Trigger CI From d752ce9452bf6464b07493a94b518cf0b25cea0c Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 9 Sep 2025 15:31:29 +0200 Subject: [PATCH 16/16] Trigger CI