From 02bf5c917c896bc5481a219a1474804a9e44f63b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 1 Nov 2022 14:26:09 -0700 Subject: [PATCH 1/2] Use `multiprocessing` `start_method` `"spawn"` Quoting from https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods * fork ... Note that safely forking a multithreaded process is problematic. Available on Unix only. The default on Unix. The idea to try `"spawn"` is the result of deadlock debugging. After reviewing tracebacks @jbms wrote: > This actually looks like a general fork thread-safety issues, not related to pybind11 > Using fork in a multithreaded program is generally not safe --- tests/test_gil_scoped.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index e890a7b0c8..eee27bfdba 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -144,7 +144,7 @@ def _intentional_deadlock(): ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) -SKIP_IF_DEADLOCK = True # See PR #4216 +SKIP_IF_DEADLOCK = False # See PR #4216 def _run_in_process(target, *args, **kwargs): @@ -154,7 +154,8 @@ def _run_in_process(target, *args, **kwargs): test_fn = args[0] # Do not need to wait much, 10s should be more than enough. timeout = 0.1 if test_fn is _intentional_deadlock else 10 - process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) + mp_ctx = multiprocessing.get_context("spawn") + process = mp_ctx.Process(target=target, args=args, kwargs=kwargs) process.daemon = True try: t_start = time.time() From 14e6371a42b6d37803c4fa08e56d12cc3a53729c Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 2 Nov 2022 10:40:34 -0700 Subject: [PATCH 2/2] Use `multiprocessing.set_start_method("spawn")` in conftest.py (very similar to #4306, the only difference between the two PRs is the `"spawn"` vs `"forkserver"` argument). --- tests/conftest.py | 5 +++++ tests/test_gil_scoped.py | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f5ddb9f129..3e31b03ec3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,8 @@ import contextlib import difflib import gc +import multiprocessing +import os import re import textwrap @@ -15,6 +17,9 @@ # Early diagnostic for failed imports import pybind11_tests +if os.name != "nt": + multiprocessing.set_start_method("spawn") + _long_marker = re.compile(r"([0-9])L") _hexadecimal = re.compile(r"0x[0-9a-fA-F]+") diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index eee27bfdba..931e3af213 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -154,8 +154,7 @@ def _run_in_process(target, *args, **kwargs): test_fn = args[0] # Do not need to wait much, 10s should be more than enough. timeout = 0.1 if test_fn is _intentional_deadlock else 10 - mp_ctx = multiprocessing.get_context("spawn") - process = mp_ctx.Process(target=target, args=args, kwargs=kwargs) + process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) process.daemon = True try: t_start = time.time()