diff --git a/src/pytask_parallel/backends.py b/src/pytask_parallel/backends.py index 9db8060..26f56e2 100644 --- a/src/pytask_parallel/backends.py +++ b/src/pytask_parallel/backends.py @@ -84,6 +84,8 @@ def _get_thread_pool_executor(n_workers: int) -> Executor: class ParallelBackend(Enum): """Choices for parallel backends.""" + NONE = "none" + CUSTOM = "custom" DASK = "dask" LOKY = "loky" diff --git a/src/pytask_parallel/build.py b/src/pytask_parallel/build.py index 11a1a0b..b54bd08 100644 --- a/src/pytask_parallel/build.py +++ b/src/pytask_parallel/build.py @@ -26,7 +26,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: ["--parallel-backend"], type=EnumChoice(ParallelBackend), help="Backend for the parallelization.", - default=ParallelBackend.LOKY, + default=ParallelBackend.NONE, ), ] cli.commands["build"].params.extend(additional_parameters) diff --git a/src/pytask_parallel/config.py b/src/pytask_parallel/config.py index c60c828..ffd70a6 100644 --- a/src/pytask_parallel/config.py +++ b/src/pytask_parallel/config.py @@ -21,9 +21,6 @@ def pytask_parse_config(config: dict[str, Any]) -> None: """Parse the configuration.""" __tracebackhide__ = True - if config["n_workers"] == "auto": - config["n_workers"] = max(os.cpu_count() - 1, 1) - try: config["parallel_backend"] = ParallelBackend(config["parallel_backend"]) except ValueError: @@ -33,6 +30,13 @@ def pytask_parse_config(config: dict[str, Any]) -> None: ) raise ValueError(msg) from None + if config["n_workers"] == "auto": + config["n_workers"] = max(os.cpu_count() - 1, 1) + + # If more than one worker is used, and no backend is set, use loky. + if config["n_workers"] > 1 and config["parallel_backend"] == ParallelBackend.NONE: + config["parallel_backend"] = ParallelBackend.LOKY + config["delay"] = 0.1 @@ -43,6 +47,9 @@ def pytask_post_parse(config: dict[str, Any]) -> None: if config["pdb"] or config["trace"] or config["dry_run"]: return + if config["parallel_backend"] == ParallelBackend.NONE: + return + # Register parallel execute and logging hook. config["pm"].register(logging) config["pm"].register(execute)