From 87917a22e19c3f37a97b038517d5cc9ca0e06706 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 6 Apr 2024 13:09:42 +0200 Subject: [PATCH 1/2] Implement a none backend as default. --- src/pytask_parallel/backends.py | 2 ++ src/pytask_parallel/build.py | 2 +- src/pytask_parallel/config.py | 10 +++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/pytask_parallel/backends.py b/src/pytask_parallel/backends.py index 9db8060..26f56e2 100644 --- a/src/pytask_parallel/backends.py +++ b/src/pytask_parallel/backends.py @@ -84,6 +84,8 @@ def _get_thread_pool_executor(n_workers: int) -> Executor: class ParallelBackend(Enum): """Choices for parallel backends.""" + NONE = "none" + CUSTOM = "custom" DASK = "dask" LOKY = "loky" diff --git a/src/pytask_parallel/build.py b/src/pytask_parallel/build.py index 11a1a0b..b54bd08 100644 --- a/src/pytask_parallel/build.py +++ b/src/pytask_parallel/build.py @@ -26,7 +26,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: ["--parallel-backend"], type=EnumChoice(ParallelBackend), help="Backend for the parallelization.", - default=ParallelBackend.LOKY, + default=ParallelBackend.NONE, ), ] cli.commands["build"].params.extend(additional_parameters) diff --git a/src/pytask_parallel/config.py b/src/pytask_parallel/config.py index c60c828..fc799d4 100644 --- a/src/pytask_parallel/config.py +++ b/src/pytask_parallel/config.py @@ -21,9 +21,6 @@ def pytask_parse_config(config: dict[str, Any]) -> None: """Parse the configuration.""" __tracebackhide__ = True - if config["n_workers"] == "auto": - config["n_workers"] = max(os.cpu_count() - 1, 1) - try: config["parallel_backend"] = ParallelBackend(config["parallel_backend"]) except ValueError: @@ -33,6 +30,13 @@ def pytask_parse_config(config: dict[str, Any]) -> None: ) raise ValueError(msg) from None + if config["n_workers"] == "auto": + config["n_workers"] = max(os.cpu_count() - 1, 1) + + # If more than one worker is used, and no backend is set, use loky. + if config["n_workers"] > 1 and config["parallel_backend"] == ParallelBackend.NONE: + config["parallel_backend"] = ParallelBackend.LOKY + config["delay"] = 0.1 From 732f0339d15071054d212a5b7acd192a5bd81eee Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 7 Apr 2024 14:02:56 +0200 Subject: [PATCH 2/2] Fix tests. --- src/pytask_parallel/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pytask_parallel/config.py b/src/pytask_parallel/config.py index fc799d4..ffd70a6 100644 --- a/src/pytask_parallel/config.py +++ b/src/pytask_parallel/config.py @@ -47,6 +47,9 @@ def pytask_post_parse(config: dict[str, Any]) -> None: if config["pdb"] or config["trace"] or config["dry_run"]: return + if config["parallel_backend"] == ParallelBackend.NONE: + return + # Register parallel execute and logging hook. config["pm"].register(logging) config["pm"].register(execute)