diff --git a/docs/source/changes.md b/docs/source/changes.md index 31eef098..5b9038fd 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -23,6 +23,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`528` improves the codecov setup and coverage. - {pull}`535` reenables and fixes tests with Jupyter. - {pull}`536` allows partialed functions to be task functions. +- {pull}`539` implements the {confval}`hook_module` configuration value and + `--hook-module` commandline option to register hooks. - {pull}`540` changes the CLI entry-point and allow `pytask.build(tasks=task_func)` as the signatures suggested. - {pull}`542` refactors the plugin manager. diff --git a/docs/source/how_to_guides/how_to_write_a_plugin.md b/docs/source/how_to_guides/extending_pytask.md similarity index 56% rename from docs/source/how_to_guides/how_to_write_a_plugin.md rename to docs/source/how_to_guides/extending_pytask.md index 6fc024a0..0d65c308 100644 --- a/docs/source/how_to_guides/how_to_write_a_plugin.md +++ b/docs/source/how_to_guides/extending_pytask.md @@ -1,11 +1,71 @@ -# How to write a plugin +# Extending pytask -Since pytask is based on pluggy, it is extensible. In this section, you will learn some -key concepts you need to know to write a plugin. It won't deal with pluggy in detail, -but if you are interested feel free to read [pluggy](../explanations/pluggy.md). A quick -look at the first paragraphs might be useful nonetheless. +pytask can be extended since it is built upon +[pluggy](https://pluggy.readthedocs.io/en/latest/), a plugin system for Python. -## Preparation +How does it work? Throughout the execution, pytask arrives at entrypoints, called hook +functions. When pytask calls a hook function it loops through hook implementations and +each hook implementation can alter the result of the entrypoint. + +The full list of hook functions is specified in {doc}`../reference_guides/hookspecs`. + +More general information about pluggy can be found in its +[documentation](https://pluggy.readthedocs.io/en/latest/). + +There are two ways to add new hook implementations. + +1. Using the {option}`pytask build --hook-module` commandline option or the + {confval}`hook_module` configuration value. +1. Packaging your plugin as a Python package to publish and share it. + +(hook-module)= + +## Using `--hook-module` and `hook_module` + +The easiest and quickest way to extend pytask is to create a module, for example, +`hooks.py` and register it temporarily via the commandline option or permanently via the +configuration. + +```console +pytask --hook-module hooks.py +``` + +or + +```toml +[tool.pytask.ini_options] +hook_module = ["hooks.py"] +``` + +The value can be a path. If the path is relative it is assumed to be relative to the +configuration file or relative to the current working directory as a fallback. + +The value can also be a module name. For example, if `hooks.py` lies your projects +package called `myproject` which is importable, then, you can also use + +```toml +[tool.pytask.ini_options] +hook_module = ["myproject.hooks"] +``` + +In `hooks.py` we can add another commandline option to `pytask build` by providing an +addition hook implementation for the hook specification +{func}`~_pytask.hookspecs.pytask_extend_command_line_interface`. + +```python +import click +from _pytask.pluginmanager import hookimpl + + +@hookimpl +def pytask_extend_command_line_interface(cli): + """Add parameters to the command line interface.""" + cli.commands["build"].params.append(click.Option(["--hello"])) +``` + +## Packaging a plugin + +### Preparation Before you start implementing your plugin, the following notes may help you. @@ -24,11 +84,11 @@ Before you start implementing your plugin, the following notes may help you. for your plugin to get feedback from other developers. Your proposal should be concise and explain what problem you want to solve and how. -## Writing your plugin +### Writing your plugin This section explains some steps which are required for all plugins. -### Set up the setuptools entry-point +#### Set up the setuptools entry-point pytask discovers plugins via `setuptools` entry-points. Following the approach advocated for by [setuptools_scm](https://github.com/pypa/setuptools_scm), the entry-point is @@ -65,7 +125,7 @@ For a complete example with `setuptools_scm` and `pyproject.toml` see the The entry-point for pytask is called `"pytask"` and points to a module called `pytask_plugin.plugin`. -### `plugin.py` +#### `plugin.py` `plugin.py` is the entrypoint for pytask to your package. You can put all of your hook implementations in this module, but it is recommended to imitate the structure of pytask diff --git a/docs/source/how_to_guides/index.md b/docs/source/how_to_guides/index.md index 1f8e8f74..0babca6a 100644 --- a/docs/source/how_to_guides/index.md +++ b/docs/source/how_to_guides/index.md @@ -20,7 +20,7 @@ how_to_influence_build_order hashing_inputs_of_tasks using_task_returns writing_custom_nodes -how_to_write_a_plugin +extending_pytask the_data_catalog ``` diff --git a/docs/source/reference_guides/configuration.md b/docs/source/reference_guides/configuration.md index eecc42ce..fed10f86 100644 --- a/docs/source/reference_guides/configuration.md +++ b/docs/source/reference_guides/configuration.md @@ -97,6 +97,21 @@ editor_url_scheme = "no_link" ```` +````{confval} hook_module + +Register additional modules containing hook implementations. + +```toml +hook_modules = ["myproject.hooks", "hooks.py"] +``` + +You can use module names and paths as values. Relative paths are assumed to be relative +to the configuration file or the current working directory. + +{ref}`This how-to guide ` has more information. + +```` + ````{confval} ignore pytask can ignore files and directories and exclude some tasks or reduce the duration of diff --git a/docs/source/tutorials/plugins.md b/docs/source/tutorials/plugins.md index 4a8c8b35..a7eaba42 100644 --- a/docs/source/tutorials/plugins.md +++ b/docs/source/tutorials/plugins.md @@ -19,7 +19,7 @@ You can find plugins in many places. - Search on [anaconda.org](https://anaconda.org/search?q=pytask) or [prefix.dev](https://prefix.dev) for related packages. -## How to implement your plugin +## How to extend pytask -Follow the {doc}`guide on writing a plugin <../how_to_guides/how_to_write_a_plugin>` to -write your plugin. +Follow the {doc}`guide on extending pytask <../how_to_guides/extending_pytask>` to add +your own hook implementations or write your plugin. diff --git a/src/_pytask/click.py b/src/_pytask/click.py index 478eb335..35e9fd9b 100644 --- a/src/_pytask/click.py +++ b/src/_pytask/click.py @@ -1,15 +1,21 @@ """Contains code related to click.""" from __future__ import annotations -import enum import inspect +from enum import Enum from gettext import gettext as _ +from gettext import ngettext from typing import Any from typing import ClassVar +from typing import TYPE_CHECKING import click from _pytask import __version__ as version from _pytask.console import console +from click import Choice +from click import Command +from click import Context +from click import Parameter from click.parser import split_opt from click_default_group import DefaultGroup from rich.highlighter import RegexHighlighter @@ -17,11 +23,14 @@ from rich.table import Table from rich.text import Text +if TYPE_CHECKING: + from collections.abc import Sequence + __all__ = ["ColoredCommand", "ColoredGroup", "EnumChoice"] -class EnumChoice(click.Choice): +class EnumChoice(Choice): """An enum-based choice type. The implementation is copied from https://github.com/pallets/click/pull/2210 and @@ -35,17 +44,15 @@ class EnumChoice(click.Choice): """ - def __init__(self, enum_type: type[enum.Enum], case_sensitive: bool = True) -> None: + def __init__(self, enum_type: type[Enum], case_sensitive: bool = True) -> None: super().__init__( choices=[element.value for element in enum_type], case_sensitive=case_sensitive, ) self.enum_type = enum_type - def convert( - self, value: Any, param: click.Parameter | None, ctx: click.Context | None - ) -> Any: - if isinstance(value, enum.Enum): + def convert(self, value: Any, param: Parameter | None, ctx: Context | None) -> Any: + if isinstance(value, Enum): value = value.value value = super().convert(value=value, param=param, ctx=ctx) if value is None: @@ -68,7 +75,7 @@ class ColoredGroup(DefaultGroup): def format_help( self: DefaultGroup, - ctx: click.Context, + ctx: Context, formatter: Any, # noqa: ARG002 ) -> None: """Format the help text.""" @@ -114,12 +121,62 @@ def format_help( ) -class ColoredCommand(click.Command): +def _iter_params_for_processing( + invocation_order: Sequence[Parameter], declaration_order: Sequence[Parameter] +) -> list[Parameter]: + def sort_key(item: Parameter) -> tuple[bool, float]: + # Hardcode the order of the config and paths parameters so that they are always + # processed first even if other eager parameters are chosen. The rest follows + # https://click.palletsprojects.com/en/8.1.x/advanced/#callback-evaluation-order. + if item.name == "paths": + return False, -3 + + if item.name == "config": + return False, -2 + + if item.name == "hook_module": + return False, -1 + + try: + idx: float = invocation_order.index(item) + except ValueError: + idx = float("inf") + + return not item.is_eager, idx + + return sorted(declaration_order, key=sort_key) + + +class ColoredCommand(Command): """A command with colored help pages.""" + def parse_args(self, ctx: Context, args: list[str]) -> list[str]: + if not args and self.no_args_is_help and not ctx.resilient_parsing: + click.echo(ctx.get_help(), color=ctx.color) + ctx.exit() + + parser = self.make_parser(ctx) + opts, args, param_order = parser.parse_args(args=args) + + for param in _iter_params_for_processing(param_order, self.get_params(ctx)): + value, args = param.handle_parse_result(ctx, opts, args) + + if args and not ctx.allow_extra_args and not ctx.resilient_parsing: + ctx.fail( + ngettext( + "Got unexpected extra argument ({args})", + "Got unexpected extra arguments ({args})", + len(args), + ).format(args=" ".join(map(str, args))) + ) + + ctx.args = args + ctx._opt_prefixes.update(parser._opt_prefixes) + return args + def format_help( - self: click.Command, - ctx: click.Context, + self: Command, + ctx: Context, formatter: Any, # noqa: ARG002 ) -> None: """Format the help text.""" @@ -142,9 +199,7 @@ def format_help( ) -def _print_options( - group_or_command: click.Command | DefaultGroup, ctx: click.Context -) -> None: +def _print_options(group_or_command: Command | DefaultGroup, ctx: Context) -> None: """Print options formatted with a table in a panel.""" highlighter = _OptionHighlighter() @@ -195,7 +250,7 @@ def _print_options( def _format_help_text( # noqa: C901, PLR0912, PLR0915 - param: click.Parameter, ctx: click.Context + param: Parameter, ctx: Context ) -> Text: """Format the help of a click parameter. @@ -264,7 +319,7 @@ def _format_help_text( # noqa: C901, PLR0912, PLR0915 and not default_value ): default_string = "" - elif isinstance(default_value, enum.Enum): + elif isinstance(default_value, Enum): default_string = str(default_value.value) else: default_string = str(default_value) diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index 36e547fe..555327fc 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -1,16 +1,25 @@ """Contains common parameters for the commands of the command line interface.""" from __future__ import annotations +import importlib.util from pathlib import Path +from typing import Iterable +from typing import TYPE_CHECKING import click from _pytask.config_utils import set_defaults_from_config +from _pytask.path import import_path from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import register_hook_impls_from_modules +from _pytask.pluginmanager import storage from click import Context from sqlalchemy.engine import make_url from sqlalchemy.engine import URL from sqlalchemy.exc import ArgumentError +if TYPE_CHECKING: + from pluggy import PluginManager + _CONFIG_OPTION = click.Option( ["-c", "--config"], @@ -96,20 +105,84 @@ def _database_url_callback( _DATABASE_URL_OPTION = click.Option( ["--database-url"], type=str, - help=("Url to the database."), + help="Url to the database.", default=None, show_default="sqlite:///.../.pytask/pytask.sqlite3", callback=_database_url_callback, ) +def _hook_module_callback( + ctx: Context, + name: str, # noqa: ARG001 + value: tuple[str, ...], +) -> Iterable[str | Path]: + """Register the user's hook modules from the configuration file.""" + if not value: + return value + + parsed_modules = [] + for module_name in value: + if module_name.endswith(".py"): + path = Path(module_name) + if ctx.params["config"]: + path = ctx.params["config"].parent.joinpath(path).resolve() + else: + path = Path.cwd().joinpath(path).resolve() + + if not path.exists(): + msg = ( + f"The hook module {path} does not exist. " + "Please provide a valid path." + ) + raise click.BadParameter(msg) + module = import_path(path, ctx.params["root"]) + parsed_modules.append(module.__name__) + else: + spec = importlib.util.find_spec(module_name) + if spec is None: + msg = ( + f"The hook module {module_name!r} is not importable. " + "Please provide a valid module name." + ) + raise click.BadParameter(msg) + parsed_modules.append(module_name) + + # If there are hook modules, we register a hook implementation to add them. + # ``pytask_add_hooks`` is a historic hook specification, so even command line + # options can be added. + if parsed_modules: + + class HookModule: + @staticmethod + @hookimpl + def pytask_add_hooks(pm: PluginManager) -> None: + """Add hooks.""" + register_hook_impls_from_modules(pm, parsed_modules) + + pm = storage.get() + pm.register(HookModule) + + return parsed_modules + + +_HOOK_MODULE_OPTION = click.Option( + ["--hook-module"], + type=str, + help="Path to a Python module that contains hook implementations.", + multiple=True, + is_eager=True, + callback=_hook_module_callback, +) + + @hookimpl(trylast=True) def pytask_extend_command_line_interface(cli: click.Group) -> None: """Register general markers.""" for command in ("build", "clean", "collect", "dag", "profile"): - cli.commands[command].params.extend([_PATH_ARGUMENT, _DATABASE_URL_OPTION]) + cli.commands[command].params.extend((_PATH_ARGUMENT, _DATABASE_URL_OPTION)) for command in ("build", "clean", "collect", "dag", "markers", "profile"): - cli.commands[command].params.append(_CONFIG_OPTION) + cli.commands[command].params.extend((_CONFIG_OPTION, _HOOK_MODULE_OPTION)) for command in ("build", "clean", "collect", "profile"): cli.commands[command].params.extend([_IGNORE_OPTION, _EDITOR_URL_SCHEME_OPTION]) for command in ("build",): diff --git a/tests/conftest.py b/tests/conftest.py index 756200e7..ee77081b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,10 +7,10 @@ from typing import Any import pytest +from _pytask.pluginmanager import storage from click.testing import CliRunner from packaging import version from pytask import console -from pytask import storage @pytest.fixture(autouse=True) diff --git a/tests/test_hook_module.py b/tests/test_hook_module.py new file mode 100644 index 00000000..24784c1d --- /dev/null +++ b/tests/test_hook_module.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap + +import pytest +from pytask import ExitCode + + +@pytest.mark.parametrize( + "module_name", + [ + pytest.param( + True, + marks=pytest.mark.xfail( + sys.platform == "win32" and "CI" in os.environ, + reason="pytask is not found in subprocess", + strict=True, + ), + ), + False, + ], +) +def test_add_new_hook_via_cli(tmp_path, module_name): + hooks = """ + import click + from pytask import hookimpl + + @hookimpl + def pytask_extend_command_line_interface(cli): + print("Hello World!") + cli.commands["build"].params.append(click.Option(["--new-option"])) + """ + tmp_path.joinpath("hooks").mkdir() + tmp_path.joinpath("hooks", "hooks.py").write_text(textwrap.dedent(hooks)) + + if module_name: + args = ( + "python", + "-m", + "pytask", + "build", + "--hook-module", + "hooks.hooks", + "--help", + ) + else: + args = ("pytask", "build", "--hook-module", "hooks/hooks.py", "--help") + + result = subprocess.run(args, cwd=tmp_path, capture_output=True, check=True) + + assert result.returncode == ExitCode.OK + assert "--new-option" in result.stdout.decode() + + +@pytest.mark.parametrize( + "module_name", + [ + pytest.param( + True, + marks=pytest.mark.xfail( + sys.platform == "win32" and "CI" in os.environ, + reason="pytask is not found in subprocess", + strict=True, + ), + ), + False, + ], +) +def test_add_new_hook_via_config(tmp_path, module_name): + tmp_path.joinpath("pyproject.toml").write_text( + "[tool.pytask.ini_options]\nhook_module = ['hooks/hooks.py']" + ) + + hooks = """ + import click + from pytask import hookimpl + + @hookimpl + def pytask_extend_command_line_interface(cli): + cli.commands["build"].params.append(click.Option(["--new-option"])) + """ + tmp_path.joinpath("hooks").mkdir() + tmp_path.joinpath("hooks", "hooks.py").write_text(textwrap.dedent(hooks)) + + if module_name: + args = ("python", "-m", "pytask", "build", "--help") + else: + args = ("pytask", "build", "--help") + + result = subprocess.run( + args, + cwd=tmp_path, + capture_output=True, + check=True, + ) + assert result.returncode == ExitCode.OK + assert "--new-option" in result.stdout.decode() + + +def test_error_when_hook_module_path_does_not_exist(tmp_path): + result = subprocess.run( # noqa: PLW1510 + ("pytask", "build", "--hook-module", "hooks.py", "--help"), + cwd=tmp_path, + capture_output=True, + ) + assert result.returncode == ExitCode.CONFIGURATION_FAILED + assert b"Error: Invalid value for '--hook-module'" in result.stderr + + +def test_error_when_hook_module_module_does_not_exist(tmp_path): + result = subprocess.run( # noqa: PLW1510 + ("pytask", "build", "--hook-module", "hooks", "--help"), + cwd=tmp_path, + capture_output=True, + ) + assert result.returncode == ExitCode.CONFIGURATION_FAILED + assert b"Error: Invalid value for '--hook-module':" in result.stderr + + +def test_error_when_hook_module_is_no_iterable(tmp_path): + tmp_path.joinpath("pyproject.toml").write_text( + "[tool.pytask.ini_options]\nhook_module = 'hooks'" + ) + result = subprocess.run( # noqa: PLW1510 + ("pytask", "build", "--help"), cwd=tmp_path, capture_output=True + ) + assert result.returncode == ExitCode.CONFIGURATION_FAILED + assert b"Error: Invalid value for '--hook-module':" in result.stderr