From d606d36f617e2320d18f4c9da6e7fa2bef5bd682 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 12 Mar 2024 21:02:17 +0100 Subject: [PATCH 01/15] TEMP. --- .pre-commit-config.yaml | 11 +++++ pyproject.toml | 1 + src/_pytask/build.py | 88 ++++++++++++++++++------------------ src/_pytask/cli.py | 18 ++++++-- src/_pytask/debugging.py | 61 ++++++++++++------------- src/_pytask/pluginmanager.py | 38 ++++++++-------- 6 files changed, 118 insertions(+), 99 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 026a3f29..2c512bb2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,16 @@ repos: hooks: - id: refurb args: [--ignore, FURB126] + additional_dependencies: [ + attrs>=21.3.0, + click, + optree, + pluggy>=1.3.0, + rich, + sqlalchemy>2, + typed-settings, + types-setuptools, + ] - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.8.0 hooks: @@ -54,6 +64,7 @@ repos: pluggy>=1.3.0, rich, sqlalchemy>2, + typed-settings, types-setuptools, ] pass_filenames: false diff --git a/pyproject.toml b/pyproject.toml index 9a0ef132..cab1e632 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,6 +174,7 @@ disallow_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true warn_unused_ignores = true +plugins = ["typed_settings.mypy"] [[tool.mypy.overrides]] diff --git a/src/_pytask/build.py b/src/_pytask/build.py index e93c9ada..958fdf79 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -12,12 +12,11 @@ from typing import Iterable from typing import Literal -import click +import typed_settings as ts from rich.traceback import Traceback from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture -from _pytask.click import ColoredCommand from _pytask.config_utils import find_project_root_and_config from _pytask.config_utils import read_config from _pytask.console import console @@ -39,13 +38,54 @@ if TYPE_CHECKING: from typing import NoReturn + import click + from _pytask.node_protocols import PTask +@ts.settings +class Base: + ... + # debug_pytask: bool = ts.option( + # default=False, + # help="Trace all function calls in the plugin framework.", + # ) + # stop_after_first_failure: bool = ts.option( + # default=False, + # click={"param_decls": ("-x", "--stop-after-first-failure"), "is_flag": True}, + # help="Stop after the first failure.", + # ) + # max_failures: float = ts.option( + # default=float("inf"), + # click={"param_decls": ("--max-failures",)}, + # help="Stop after some failures.", + # ) + # show_errors_immediately: bool = ts.option( + # default=False, + # click={"param_decls": ("--show-errors-immediately",), "is_flag": True}, + # help="Show errors with tracebacks as soon as the task fails.", + # ) + # show_traceback: bool = ts.option( + # default=True, + # click={"param_decls": ("--show-traceback", "--show-no-traceback")}, + # help="Choose whether tracebacks should be displayed or not.", + # ) + # dry_run: bool = ts.option( + # default=False, + # click={"param_decls": ("--dry-run",), "is_flag": True}, + # help="Perform a dry-run.", + # ) + # force: bool = ts.option( + # default=False, + # click={"param_decls": ("-f", "--force"), "is_flag": True}, + # help="Execute a task even if it succeeded successfully before.", + # ) + + @hookimpl(tryfirst=True) def pytask_extend_command_line_interface(cli: click.Group) -> None: """Extend the command line interface.""" - cli.add_command(build_command) + cli["build"] = {"command": build_command, "base": Base, "options": {}} @hookimpl @@ -288,48 +328,6 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 return session -@click.command(cls=ColoredCommand, name="build") -@click.option( - "--debug-pytask", - is_flag=True, - default=False, - help="Trace all function calls in the plugin framework.", -) -@click.option( - "-x", - "--stop-after-first-failure", - is_flag=True, - default=False, - help="Stop after the first failure.", -) -@click.option( - "--max-failures", - type=click.FloatRange(min=1), - default=float("inf"), - help="Stop after some failures.", -) -@click.option( - "--show-errors-immediately", - is_flag=True, - default=False, - help="Show errors with tracebacks as soon as the task fails.", -) -@click.option( - "--show-traceback/--show-no-traceback", - type=bool, - default=True, - help="Choose whether tracebacks should be displayed or not.", -) -@click.option( - "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry-run." -) -@click.option( - "-f", - "--force", - is_flag=True, - default=False, - help="Execute a task even if it succeeded successfully before.", -) def build_command(**raw_config: Any) -> NoReturn: """Collect tasks, execute them and report the results. diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index 13046646..f8a2755a 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -5,7 +5,9 @@ from typing import Any import click +import typed_settings as ts from packaging.version import parse as parse_version +from typed_settings.cli_click import OptionGroupFactory from _pytask.click import ColoredGroup from _pytask.pluginmanager import storage @@ -25,9 +27,10 @@ def _extend_command_line_interface(cli: click.Group) -> click.Group: """Add parameters from plugins to the commandline interface.""" pm = storage.create() - pm.hook.pytask_extend_command_line_interface.call_historic(kwargs={"cli": cli}) - _sort_options_for_each_command_alphabetically(cli) - return cli + commands = {} + pm.hook.pytask_extend_command_line_interface.call_historic(kwargs={"cli": commands}) + # _sort_options_for_each_command_alphabetically(cli) + return commands def _sort_options_for_each_command_alphabetically(cli: click.Group) -> None: @@ -49,7 +52,14 @@ def cli() -> None: """Manage your tasks with pytask.""" -_extend_command_line_interface(cli) +commands = _extend_command_line_interface(cli) + + +for name, data in commands.items(): + settings = ts.combine("settings", data["base"], data["options"]) + command = data["command"] + command = ts.click_options(settings, "build", decorator_factory=OptionGroupFactory())(command) + cli.add_command(click.command(name=name)(command)) DEFAULTS_FROM_CLI = { diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index 6929c312..776e8aeb 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -11,6 +11,7 @@ from typing import Generator import click +import typed_settings as ts from _pytask.console import console from _pytask.node_protocols import PTask @@ -29,37 +30,6 @@ from _pytask.session import Session -@hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: - """Extend command line interface.""" - additional_parameters = [ - click.Option( - ["--pdb"], - help="Start the interactive debugger on errors.", - is_flag=True, - default=False, - ), - click.Option( - ["--trace"], - help="Enter debugger in the beginning of each task.", - is_flag=True, - default=False, - ), - click.Option( - ["--pdbcls"], - help=( - "Start a custom debugger on errors. For example: " - "--pdbcls=IPython.terminal.debugger:TerminalPdb" - ), - type=str, - default=None, - metavar="module_name:class_name", - callback=_pdbcls_callback, - ), - ] - cli.commands["build"].params.extend(additional_parameters) - - def _pdbcls_callback( ctx: click.Context, # noqa: ARG001 name: str, # noqa: ARG001 @@ -78,6 +48,35 @@ def _pdbcls_callback( raise click.BadParameter(message) +@ts.settings +class Debugging: + pdb: bool = ts.option( + default=False, + click={"param_decls": ("--pdb",)}, + help="Start the interactive debugger on errors.", + ) + pdbcls: str = ts.option( + default="", + click={ + "param_decls": ("--pdb-cls",), + "metavar": "module_name:class_name", + "callback": _pdbcls_callback, + }, + help="Start a custom debugger on errors. For example: --pdbcls=IPython.terminal.debugger:TerminalPdb", + ) + trace: bool = ts.option( + default=False, + click={"param_decls": ("--trace",)}, + help="Enter debugger in the beginning of each task.", + ) + + +@hookimpl +def pytask_extend_command_line_interface(cli: click.Group) -> None: + """Extend command line interface.""" + cli["build"]["options"]["debugging"] = Debugging() + + @hookimpl(trylast=True) def pytask_post_parse(config: dict[str, Any]) -> None: """Post parse the configuration. diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index 3159915c..2a10d25b 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -37,26 +37,26 @@ def pytask_add_hooks(pm: PluginManager) -> None: """Add hooks.""" builtin_hook_impl_modules = ( "_pytask.build", - "_pytask.capture", - "_pytask.clean", - "_pytask.collect", - "_pytask.collect_command", - "_pytask.config", - "_pytask.dag", - "_pytask.dag_command", - "_pytask.database", + # "_pytask.capture", + # "_pytask.clean", + # "_pytask.collect", + # "_pytask.collect_command", + # "_pytask.config", + # "_pytask.dag", + # "_pytask.dag_command", + # "_pytask.database", "_pytask.debugging", - "_pytask.execute", - "_pytask.live", - "_pytask.logging", - "_pytask.mark", - "_pytask.nodes", - "_pytask.parameters", - "_pytask.persist", - "_pytask.profile", - "_pytask.skipping", - "_pytask.task", - "_pytask.warnings", + # "_pytask.execute", + # "_pytask.live", + # "_pytask.logging", + # "_pytask.mark", + # "_pytask.nodes", + # "_pytask.parameters", + # "_pytask.persist", + # "_pytask.profile", + # "_pytask.skipping", + # "_pytask.task", + # "_pytask.warnings", ) register_hook_impls_from_modules(pm, builtin_hook_impl_modules) From 31c863182a6079ffd5340635c4941a4da66d7458 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 23 Mar 2024 10:52:54 +0100 Subject: [PATCH 02/15] Progress on build command. --- .pre-commit-config.yaml | 5 - pyproject.toml | 7 + src/_pytask/build.py | 158 +++++++-------- src/_pytask/capture.py | 76 ++++--- src/_pytask/clean.py | 68 ++++--- src/_pytask/cli.py | 42 ++-- src/_pytask/click.py | 2 +- src/_pytask/collect_command.py | 28 +-- src/_pytask/config_utils.py | 66 ++----- src/_pytask/dag_command.py | 80 ++++---- src/_pytask/database.py | 52 +++++ src/_pytask/debugging.py | 7 +- src/_pytask/hookspecs.py | 6 +- src/_pytask/live.py | 41 ++-- src/_pytask/logging.py | 27 ++- src/_pytask/mark/__init__.py | 55 +++--- src/_pytask/parameters.py | 186 +++++++----------- src/_pytask/pluginmanager.py | 30 +-- src/_pytask/profile.py | 18 +- src/_pytask/settings.py | 173 ++++++++++++++++ src/_pytask/warnings.py | 34 ++-- tests/test_capture.py | 5 +- .../test_functional_interface.ipynb | 7 +- ...functional_interface_w_relative_path.ipynb | 7 +- tests/test_jupyter/test_task_generator.ipynb | 9 +- 25 files changed, 682 insertions(+), 507 deletions(-) create mode 100644 src/_pytask/settings.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 507f4844..650a8439 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -88,11 +88,6 @@ repos: mdformat-pyproject, ] files: (docs/.) -- repo: https://github.com/nbQA-dev/nbQA - rev: 1.8.4 - hooks: - - id: nbqa-mypy - args: [--ignore-missing-imports] - repo: https://github.com/kynan/nbstripout rev: 0.7.1 hooks: diff --git a/pyproject.toml b/pyproject.toml index cab1e632..55a8b460 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -210,3 +210,10 @@ exclude_also = [ [tool.mdformat] wrap = 88 end_of_line = "keep" + + +# [tool.pytask] +# debug_pytask = false + +[tool.pytask.ini_options] +debug_pytask = true diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 958fdf79..bb703422 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -5,7 +5,6 @@ import json import sys from contextlib import suppress -from pathlib import Path from typing import TYPE_CHECKING from typing import Any from typing import Callable @@ -17,8 +16,7 @@ from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture -from _pytask.config_utils import find_project_root_and_config -from _pytask.config_utils import read_config +from _pytask.config_utils import consolidate_settings_and_arguments from _pytask.console import console from _pytask.dag import create_dag from _pytask.exceptions import CollectionError @@ -31,61 +29,71 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.shared import parse_paths -from _pytask.shared import to_list +from _pytask.settings import SettingsBuilder +from _pytask.settings import create_settings_loaders +from _pytask.settings import update_settings from _pytask.traceback import remove_internal_traceback_frames_from_exc_info if TYPE_CHECKING: + from pathlib import Path from typing import NoReturn - import click - from _pytask.node_protocols import PTask @ts.settings class Base: - ... - # debug_pytask: bool = ts.option( - # default=False, - # help="Trace all function calls in the plugin framework.", - # ) - # stop_after_first_failure: bool = ts.option( - # default=False, - # click={"param_decls": ("-x", "--stop-after-first-failure"), "is_flag": True}, - # help="Stop after the first failure.", - # ) - # max_failures: float = ts.option( - # default=float("inf"), - # click={"param_decls": ("--max-failures",)}, - # help="Stop after some failures.", - # ) - # show_errors_immediately: bool = ts.option( - # default=False, - # click={"param_decls": ("--show-errors-immediately",), "is_flag": True}, - # help="Show errors with tracebacks as soon as the task fails.", - # ) - # show_traceback: bool = ts.option( - # default=True, - # click={"param_decls": ("--show-traceback", "--show-no-traceback")}, - # help="Choose whether tracebacks should be displayed or not.", - # ) - # dry_run: bool = ts.option( - # default=False, - # click={"param_decls": ("--dry-run",), "is_flag": True}, - # help="Perform a dry-run.", - # ) - # force: bool = ts.option( - # default=False, - # click={"param_decls": ("-f", "--force"), "is_flag": True}, - # help="Execute a task even if it succeeded successfully before.", - # ) + debug_pytask: bool = ts.option( + default=False, + click={"param_decls": ("--debug-pytask",), "is_flag": True}, + help="Trace all function calls in the plugin framework.", + ) + stop_after_first_failure: bool = ts.option( + default=False, + click={"param_decls": ("-x", "--stop-after-first-failure"), "is_flag": True}, + help="Stop after the first failure.", + ) + max_failures: float = ts.option( + default=float("inf"), + click={"param_decls": ("--max-failures",)}, + help="Stop after some failures.", + ) + show_errors_immediately: bool = ts.option( + default=False, + click={"param_decls": ("--show-errors-immediately",), "is_flag": True}, + help="Show errors with tracebacks as soon as the task fails.", + ) + show_traceback: bool = ts.option( + default=True, + click={"param_decls": ("--show-traceback", "--show-no-traceback")}, + help="Choose whether tracebacks should be displayed or not.", + ) + dry_run: bool = ts.option( + default=False, + click={"param_decls": ("--dry-run",), "is_flag": True}, + help="Perform a dry-run.", + ) + force: bool = ts.option( + default=False, + click={"param_decls": ("-f", "--force"), "is_flag": True}, + help="Execute a task even if it succeeded successfully before.", + ) + check_casing_of_paths: bool = ts.option( + default=True, + click={"param_decls": ("--check-casing-of-paths",), "hidden": True}, + ) @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend the command line interface.""" - cli["build"] = {"command": build_command, "base": Base, "options": {}} + settings_builders["build"] = SettingsBuilder( + name="build", + function=build_command, + base_settings=Base, + ) @hookimpl @@ -106,11 +114,10 @@ def pytask_unconfigure(session: Session) -> None: path.write_text(json.dumps(HashPathCache._cache)) -def build( # noqa: C901, PLR0912, PLR0913, PLR0915 +def build( # noqa: PLR0913 *, capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.FD, check_casing_of_paths: bool = True, - config: Path | None = None, database_url: str = "", debug_pytask: bool = False, disable_warnings: bool = False, @@ -127,6 +134,7 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 pdb: bool = False, pdb_cls: str = "", s: bool = False, + settings: Any = None, show_capture: Literal["no", "stdout", "stderr", "all"] | ShowCapture = ShowCapture.ALL, show_errors_immediately: bool = False, @@ -153,8 +161,6 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 The capture method for stdout and stderr. check_casing_of_paths Whether errors should be raised when file names have different casings. - config - A path to the configuration file. database_url An URL to the database that tracks the status of tasks. debug_pytask @@ -190,6 +196,8 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 ``--pdbcls=IPython.terminal.debugger:TerminalPdb`` s Shortcut for ``capture="no"``. + settings + The settings object that contains the configuration. show_capture Choose which captured output should be shown for failed tasks. show_errors_immediately @@ -221,10 +229,9 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 """ try: - raw_config = { + updates = { "capture": capture, "check_casing_of_paths": check_casing_of_paths, - "config": config, "database_url": database_url, "debug_pytask": debug_pytask, "disable_warnings": disable_warnings, @@ -254,46 +261,21 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 **kwargs, } - if "command" not in raw_config: + if settings is None: + from _pytask.cli import settings_builders + + # Create plugin manager. pm = get_plugin_manager() storage.store(pm) + + settings = ts.load_settings( + settings_builders["build"].build_settings(), create_settings_loaders() + ) else: pm = storage.get() - # If someone called the programmatic interface, we need to do some parsing. - if "command" not in raw_config: - raw_config["command"] = "build" - # Add defaults from cli. - from _pytask.cli import DEFAULTS_FROM_CLI - - raw_config = {**DEFAULTS_FROM_CLI, **raw_config} - - raw_config["paths"] = parse_paths(raw_config["paths"]) - - if raw_config["config"] is not None: - raw_config["config"] = Path(raw_config["config"]).resolve() - raw_config["root"] = raw_config["config"].parent - else: - ( - raw_config["root"], - raw_config["config"], - ) = find_project_root_and_config(raw_config["paths"]) - - if raw_config["config"] is not None: - config_from_file = read_config(raw_config["config"]) - - if "paths" in config_from_file: - paths = config_from_file["paths"] - paths = [ - raw_config["config"].parent.joinpath(path).resolve() - for path in to_list(paths) - ] - config_from_file["paths"] = paths - - raw_config = {**raw_config, **config_from_file} - - config_ = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - + settings = update_settings(settings, updates) + config_ = pm.hook.pytask_configure(pm=pm, raw_config=settings) session = Session.from_config(config_) except (ConfigurationError, Exception): @@ -328,13 +310,13 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 return session -def build_command(**raw_config: Any) -> NoReturn: +def build_command(settings: Any, **arguments: Any) -> NoReturn: """Collect tasks, execute them and report the results. The default command. pytask collects tasks from the given paths or the current working directory, executes them and reports the results. """ - raw_config["command"] = "build" - session = build(**raw_config) + settings = consolidate_settings_and_arguments(settings, arguments) + session = build(settings=settings) sys.exit(session.exit_code) diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index 951cc506..a3286d36 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -41,42 +41,45 @@ from typing import TextIO from typing import final -import click +import typed_settings as ts from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture -from _pytask.click import EnumChoice from _pytask.pluginmanager import hookimpl from _pytask.shared import convert_to_enum if TYPE_CHECKING: from _pytask.node_protocols import PTask + from _pytask.settings import SettingsBuilder + + +@ts.settings +class Capture: + """Settings for capturing.""" + + capture: CaptureMethod = ts.option( + default=CaptureMethod.FD, + click={"param_decls": ["--capture"]}, + help="Per task capturing method.", + ) + s: bool = ts.option( + default=False, + click={"param_decls": ["-s"], "is_flag": True}, + help="Shortcut for --capture=no.", + ) + show_capture: ShowCapture = ts.option( + default=ShowCapture.ALL, + click={"param_decls": ["--show-capture"]}, + help="Choose which captured output should be shown for failed tasks.", + ) @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Add CLI options for capturing output.""" - additional_parameters = [ - click.Option( - ["--capture"], - type=EnumChoice(CaptureMethod), - default=CaptureMethod.FD, - help="Per task capturing method.", - ), - click.Option( - ["-s"], - is_flag=True, - default=False, - help="Shortcut for --capture=no.", - ), - click.Option( - ["--show-capture"], - type=EnumChoice(ShowCapture), - default=ShowCapture.ALL, - help="Choose which captured output should be shown for failed tasks.", - ), - ] - cli.commands["build"].params.extend(additional_parameters) + settings_builders["build"].option_groups["capture"] = Capture() @hookimpl @@ -388,12 +391,9 @@ def __init__(self, targetfd: int) -> None: self._state = "initialized" def __repr__(self) -> str: - return "<{} {} oldfd={} _state={!r} tmpfile={!r}>".format( # noqa: UP032 - self.__class__.__name__, - self.targetfd, - self.targetfd_save, - self._state, - self.tmpfile, + return ( + f"<{self.__class__.__name__} {self.targetfd} oldfd={self.targetfd_save} " + f"_state={self._state!r} tmpfile={self.tmpfile!r}>" ) def _assert_state(self, op: str, states: tuple[str, ...]) -> None: @@ -568,15 +568,9 @@ def __init__( self.err = err def __repr__(self) -> str: - return ( # noqa: UP032 - "" - ).format( - self.out, - self.err, - self.in_, - self._state, - self._in_suspended, + return ( + f"" ) def start_capturing(self) -> None: @@ -682,8 +676,8 @@ def __init__(self, method: CaptureMethod) -> None: self._capturing: MultiCapture[str] | None = None def __repr__(self) -> str: - return ("").format( # noqa: UP032 - self._method, self._capturing + return ( + f"" ) def is_capturing(self) -> bool: diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index a4980f3e..ea357025 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -13,9 +13,9 @@ from typing import Iterable import click +import typed_settings as ts from attrs import define -from _pytask.click import ColoredCommand from _pytask.click import EnumChoice from _pytask.console import console from _pytask.exceptions import CollectionError @@ -31,6 +31,7 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session +from _pytask.settings import SettingsBuilder from _pytask.shared import to_list from _pytask.traceback import Traceback from _pytask.tree_util import tree_leaves @@ -55,10 +56,42 @@ class _CleanMode(enum.Enum): ) +@ts.settings +class Base: + directories: bool = ts.option( + default=False, + help="Remove whole directories.", + click={"is_flag": True, "param_decls": ["-d", "--directories"]}, + ) + exclude: tuple[str, ...] = ts.option( + factory=tuple, + help="A filename pattern to exclude files from the cleaning process.", + click={ + "multiple": True, + "metavar": "PATTERN", + "param_decls": ["-e", "--exclude"], + }, + ) + mode: _CleanMode = ts.option( + default=_CleanMode.DRY_RUN, + help=_HELP_TEXT_MODE, + click={"type": EnumChoice(_CleanMode), "param_decls": ["-m", "--mode"]}, + ) + quiet: bool = ts.option( + default=False, + help="Do not print the names of the removed paths.", + click={"is_flag": True, "param_decls": ["-q", "--quiet"]}, + ) + + @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend the command line interface.""" - cli.add_command(clean) + settings_builders["clean"] = SettingsBuilder( + name="clean", function=clean, base_settings=Base + ) @hookimpl @@ -67,35 +100,6 @@ def pytask_parse_config(config: dict[str, Any]) -> None: config["exclude"] = to_list(config["exclude"]) + _DEFAULT_EXCLUDE -@click.command(cls=ColoredCommand) -@click.option( - "-d", - "--directories", - is_flag=True, - default=False, - help="Remove whole directories.", -) -@click.option( - "-e", - "--exclude", - metavar="PATTERN", - multiple=True, - type=str, - help="A filename pattern to exclude files from the cleaning process.", -) -@click.option( - "--mode", - default=_CleanMode.DRY_RUN, - type=EnumChoice(_CleanMode), - help=_HELP_TEXT_MODE, -) -@click.option( - "-q", - "--quiet", - is_flag=True, - help="Do not print the names of the removed paths.", - default=False, -) def clean(**raw_config: Any) -> NoReturn: # noqa: C901, PLR0912 """Clean the provided paths by removing files unknown to pytask.""" pm = storage.get() diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index f8a2755a..87534e80 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -5,12 +5,12 @@ from typing import Any import click -import typed_settings as ts from packaging.version import parse as parse_version -from typed_settings.cli_click import OptionGroupFactory from _pytask.click import ColoredGroup from _pytask.pluginmanager import storage +from _pytask.settings import SettingsBuilder +from _pytask.settings import create_settings_loaders _CONTEXT_SETTINGS: dict[str, Any] = { "help_option_names": ("-h", "--help"), @@ -24,21 +24,14 @@ _VERSION_OPTION_KWARGS = {} -def _extend_command_line_interface(cli: click.Group) -> click.Group: +def _extend_command_line_interface() -> dict[str, SettingsBuilder]: """Add parameters from plugins to the commandline interface.""" pm = storage.create() - commands = {} - pm.hook.pytask_extend_command_line_interface.call_historic(kwargs={"cli": commands}) - # _sort_options_for_each_command_alphabetically(cli) - return commands - - -def _sort_options_for_each_command_alphabetically(cli: click.Group) -> None: - """Sort command line options and arguments for each command alphabetically.""" - for command in cli.commands: - cli.commands[command].params = sorted( - cli.commands[command].params, key=lambda x: x.opts[0].replace("-", "") - ) + settings_builders: dict[str, SettingsBuilder] = {} + pm.hook.pytask_extend_command_line_interface.call_historic( + kwargs={"settings_builders": settings_builders} + ) + return settings_builders @click.group( @@ -52,18 +45,9 @@ def cli() -> None: """Manage your tasks with pytask.""" -commands = _extend_command_line_interface(cli) - +settings_builders = _extend_command_line_interface() +settings_loaders = create_settings_loaders() -for name, data in commands.items(): - settings = ts.combine("settings", data["base"], data["options"]) - command = data["command"] - command = ts.click_options(settings, "build", decorator_factory=OptionGroupFactory())(command) - cli.add_command(click.command(name=name)(command)) - - -DEFAULTS_FROM_CLI = { - option.name: option.default - for command in cli.commands.values() - for option in command.params -} +for settings_builder in settings_builders.values(): + command = settings_builder.build_command(settings_loaders) + cli.add_command(command) diff --git a/src/_pytask/click.py b/src/_pytask/click.py index 6daab734..83ffaa1e 100644 --- a/src/_pytask/click.py +++ b/src/_pytask/click.py @@ -207,7 +207,7 @@ def _print_options(group_or_command: Command | DefaultGroup, ctx: Context) -> No options_table = Table(highlight=True, box=None, show_header=False) - for param in group_or_command.get_params(ctx): + for param in sorted(group_or_command.get_params(ctx), key=lambda x: x.name): if isinstance(param, click.Argument): continue diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 2e953e5c..f0527894 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -7,11 +7,10 @@ from typing import TYPE_CHECKING from typing import Any -import click +import typed_settings as ts from rich.text import Text from rich.tree import Tree -from _pytask.click import ColoredCommand from _pytask.console import FILE_ICON from _pytask.console import PYTHON_ICON from _pytask.console import TASK_ICON @@ -34,6 +33,7 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session +from _pytask.settings import SettingsBuilder from _pytask.tree_util import tree_leaves if TYPE_CHECKING: @@ -41,19 +41,25 @@ from typing import NoReturn +@ts.settings +class Base: + nodes: bool = ts.option( + default=False, + help="Show a task's dependencies and products.", + click={"is_flag": True}, + ) + + @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend the command line interface.""" - cli.add_command(collect) + settings_builders["collect"] = SettingsBuilder( + name="collect", function=collect, base_settings=Base + ) -@click.command(cls=ColoredCommand) -@click.option( - "--nodes", - is_flag=True, - default=False, - help="Show a task's dependencies and products.", -) def collect(**raw_config: Any | None) -> NoReturn: """Collect tasks and report information about them.""" pm = storage.get() diff --git a/src/_pytask/config_utils.py b/src/_pytask/config_utils.py index e78934d1..b90cb9e4 100644 --- a/src/_pytask/config_utils.py +++ b/src/_pytask/config_utils.py @@ -10,61 +10,29 @@ import click -from _pytask.shared import parse_paths - if sys.version_info >= (3, 11): # pragma: no cover import tomllib else: # pragma: no cover import tomli as tomllib -__all__ = ["find_project_root_and_config", "read_config", "set_defaults_from_config"] - - -def set_defaults_from_config( - context: click.Context, - param: click.Parameter, # noqa: ARG001 - value: Any, -) -> Path | None: - """Set the defaults for the command-line interface from the configuration.""" - # pytask will later walk through all configuration hooks, even the ones not related - # to this command. They might expect the defaults coming from their related - # command-line options during parsing. Here, we add their defaults to the - # configuration. - command_option_names = [option.name for option in context.command.params] - commands = context.parent.command.commands # type: ignore[union-attr] - all_defaults_from_cli = { - option.name: option.default - for name, command in commands.items() - for option in command.params - if name != context.info_name and option.name not in command_option_names - } - context.params.update(all_defaults_from_cli) - - if value: - context.params["config"] = value - context.params["root"] = context.params["config"].parent - else: - if not context.params["paths"]: - context.params["paths"] = (Path.cwd(),) - - context.params["paths"] = parse_paths(context.params["paths"]) - ( - context.params["root"], - context.params["config"], - ) = find_project_root_and_config(context.params["paths"]) - - if context.params["config"] is None: - return None - - config_from_file = read_config(context.params["config"]) - - if context.default_map is None: - context.default_map = {} - context.default_map.update(config_from_file) - context.params.update(config_from_file) - - return context.params["config"] +__all__ = ["find_project_root_and_config", "read_config"] + + +def consolidate_settings_and_arguments(settings: Any, arguments: dict[str, Any]) -> Any: + """Consolidate the settings and the values from click arguments. + + Values from the command line have precedence over the settings from the + configuration file or from environment variables. Thus, we just plug in the values + from the command line into the settings. + + """ + for key, value in arguments.items(): + # We do not want to overwrite the settings with None or empty tuples that come + # from ``multiple=True`` The default is handled by the settings class. + if value is not None and value != (): + setattr(settings, key, value) + return settings def find_project_root_and_config( diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index e41d4f37..4cc161d5 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -9,10 +9,10 @@ import click import networkx as nx +import typed_settings as ts from rich.text import Text from rich.traceback import Traceback -from _pytask.click import ColoredCommand from _pytask.click import EnumChoice from _pytask.compat import check_for_optional_program from _pytask.compat import import_optional_dependency @@ -28,6 +28,7 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session +from _pytask.settings import SettingsBuilder from _pytask.shared import parse_paths from _pytask.shared import reduce_names_of_multiple_nodes from _pytask.shared import to_list @@ -41,46 +42,43 @@ class _RankDirection(enum.Enum): RL = "RL" +@ts.settings +class Base: + layout: str = ts.option( + default="dot", + help="The layout determines the structure of the graph. Here you find an " + "overview of all available layouts: https://graphviz.org/docs/layouts.", + ) + output_path: Path = ts.option( + click={ + "type": click.Path(file_okay=True, dir_okay=False, path_type=Path), + "param_decls": ["-o", "--output-path"], + }, + default=Path("dag.pdf"), + help="The output path of the visualization. The format is inferred from the " + "file extension.", + ) + rank_direction: _RankDirection = ts.option( + default=_RankDirection.TB, + help="The direction of the directed graph. It can be ordered from top to " + "bottom, TB, left to right, LR, bottom to top, BT, or right to left, RL.", + click={ + "type": EnumChoice(_RankDirection), + "param_decls": ["-r", "--rank-direction"], + }, + ) + + @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend the command line interface.""" - cli.add_command(dag) - - -_HELP_TEXT_LAYOUT: str = ( - "The layout determines the structure of the graph. Here you find an overview of " - "all available layouts: https://graphviz.org/docs/layouts." -) - - -_HELP_TEXT_OUTPUT: str = ( - "The output path of the visualization. The format is inferred from the file " - "extension." -) - - -_HELP_TEXT_RANK_DIRECTION: str = ( - "The direction of the directed graph. It can be ordered from top to bottom, TB, " - "left to right, LR, bottom to top, BT, or right to left, RL." -) - - -@click.command(cls=ColoredCommand) -@click.option("-l", "--layout", type=str, default="dot", help=_HELP_TEXT_LAYOUT) -@click.option( - "-o", - "--output-path", - type=click.Path(file_okay=True, dir_okay=False, path_type=Path, resolve_path=True), - default="dag.pdf", - help=_HELP_TEXT_OUTPUT, -) -@click.option( - "-r", - "--rank-direction", - type=EnumChoice(_RankDirection), - help=_HELP_TEXT_RANK_DIRECTION, - default=_RankDirection.TB, -) + settings_builders["dag"] = SettingsBuilder( + name="dag", function=dag, base_settings=Base + ) + + def dag(**raw_config: Any) -> int: """Create a visualization of the project's directed acyclic graph.""" try: @@ -152,10 +150,6 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph: # If someone called the programmatic interface, we need to do some parsing. if "command" not in raw_config: raw_config["command"] = "dag" - # Add defaults from cli. - from _pytask.cli import DEFAULTS_FROM_CLI - - raw_config = {**DEFAULTS_FROM_CLI, **raw_config} raw_config["paths"] = parse_paths(raw_config["paths"]) diff --git a/src/_pytask/database.py b/src/_pytask/database.py index dfdf96b3..d00beb6e 100644 --- a/src/_pytask/database.py +++ b/src/_pytask/database.py @@ -3,13 +3,65 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING from typing import Any +import typed_settings as ts +from click import BadParameter +from click import Context +from sqlalchemy.engine import URL from sqlalchemy.engine import make_url +from sqlalchemy.exc import ArgumentError from _pytask.database_utils import create_database from _pytask.pluginmanager import hookimpl +if TYPE_CHECKING: + from _pytask.settings import SettingsBuilder + + +def _database_url_callback( + ctx: Context, # noqa: ARG001 + name: str, # noqa: ARG001 + value: str | None, +) -> URL | None: + """Check the url for the database.""" + # Since sqlalchemy v2.0.19, we need to shortcircuit here. + if value is None: + return None + + try: + return make_url(value) + except ArgumentError: + msg = ( + "The 'database_url' must conform to sqlalchemy's url standard: " + "https://docs.sqlalchemy.org/en/latest/core/engines.html#backend-specific-urls." + ) + raise BadParameter(msg) from None + + +@ts.settings +class Database: + """Settings for the database.""" + + database_url: str = ts.option( + default=None, + help="Url to the database.", + click={ + "show_default": "sqlite:///.../.pytask/pytask.sqlite3", + "callback": _database_url_callback, + }, + ) + + +@hookimpl +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: + """Extend the command line interface.""" + for settings_builder in settings_builders.values(): + settings_builder.option_groups["database"] = Database() + @hookimpl def pytask_parse_config(config: dict[str, Any]) -> None: diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index 15762628..4f959a03 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -28,6 +28,7 @@ from _pytask.capture import CaptureManager from _pytask.live import LiveManager from _pytask.session import Session + from _pytask.settings import SettingsBuilder def _pdbcls_callback( @@ -75,9 +76,11 @@ class Debugging: @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend command line interface.""" - cli["build"]["options"]["debugging"] = Debugging() + settings_builders["build"].option_groups["debugging"] = Debugging() @hookimpl(trylast=True) diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 02ce0721..3ec8100b 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -15,7 +15,6 @@ if TYPE_CHECKING: from pathlib import Path - import click from pluggy import PluginManager from _pytask.models import NodeInfo @@ -27,6 +26,7 @@ from _pytask.reports import CollectionReport from _pytask.reports import ExecutionReport from _pytask.session import Session + from _pytask.settings import SettingsBuilder hookspec = pluggy.HookspecMarker("pytask") @@ -49,7 +49,9 @@ def pytask_add_hooks(pm: PluginManager) -> None: @hookspec(historic=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend the command line interface. The hook can be used to extend the command line interface either by providing new diff --git a/src/_pytask/live.py b/src/_pytask/live.py index 3e279f71..0ab53880 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -7,7 +7,7 @@ from typing import Generator from typing import NamedTuple -import click +import typed_settings as ts from attrs import define from attrs import field from rich.box import ROUNDED @@ -28,27 +28,32 @@ from _pytask.reports import CollectionReport from _pytask.reports import ExecutionReport from _pytask.session import Session + from _pytask.settings import SettingsBuilder + + +@ts.settings +class LiveSettings: + """Settings for live display during the execution.""" + + n_entries_in_table: int = ts.option( + default=15, + click={"param_decls": ["--n-entries-in-table"]}, + help="How many entries to display in the table during the execution. " + "Tasks which are running are always displayed.", + ) + sort_table: bool = ts.option( + default=True, + click={"param_decls": ["--sort-table", "--do-not-sort-table"]}, + help="Sort the table of tasks at the end of the execution.", + ) @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend command line interface.""" - additional_parameters = [ - click.Option( - ["--n-entries-in-table"], - default=15, - type=click.IntRange(min=0), - help="How many entries to display in the table during the execution. " - "Tasks which are running are always displayed.", - ), - click.Option( - ["--sort-table/--do-not-sort-table"], - default=True, - type=bool, - help="Sort the table of tasks at the end of the execution.", - ), - ] - cli.commands["build"].params.extend(additional_parameters) + settings_builders["build"].option_groups["live"] = LiveSettings() @hookimpl diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index 545562a1..7e814cdf 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -10,8 +10,8 @@ from typing import Any from typing import NamedTuple -import click import pluggy +import typed_settings as ts from rich.text import Text import _pytask @@ -27,6 +27,7 @@ from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.session import Session + from _pytask.settings import SettingsBuilder with contextlib.suppress(ImportError): @@ -40,15 +41,27 @@ class _TimeUnit(NamedTuple): in_seconds: int -@hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: - show_locals_option = click.Option( - ["--show-locals"], - is_flag=True, +@ts.settings +class Logging: + """Settings for logging.""" + + show_capture: bool = ts.option( + default=False, + click={"param_decls": ["--show-capture"], "is_flag": True}, + help="Show the captured output of tasks.", + ) + show_locals: bool = ts.option( default=False, + click={"param_decls": ["--show-locals"], "is_flag": True}, help="Show local variables in tracebacks.", ) - cli.commands["build"].params.append(show_locals_option) + + +@hookimpl +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: + settings_builders["build"].option_groups["logging"] = Logging() @hookimpl diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 84e5d387..44b642d0 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -8,6 +8,7 @@ from typing import Any import click +import typed_settings as ts from attrs import define from rich.table import Table @@ -33,6 +34,7 @@ import networkx as nx from _pytask.node_protocols import PTask + from _pytask.settings import SettingsBuilder __all__ = [ @@ -75,33 +77,36 @@ def markers(**raw_config: Any) -> NoReturn: sys.exit(session.exit_code) +@ts.settings +class Markers: + """Settings for markers.""" + + strict_markers: bool = ts.option( + default=False, + click={"param_decls": ["--strict-markers"], "is_flag": True}, + help="Raise errors for unknown markers.", + ) + marker_expression: str = ts.option( + default="", + click={ + "param_decls": ["-m", "marker_expression"], + "metavar": "MARKER_EXPRESSION", + }, + help="Select tasks via marker expressions.", + ) + expression: str = ts.option( + default="", + click={"param_decls": ["-k", "expression"], "metavar": "EXPRESSION"}, + help="Select tasks via expressions on task ids.", + ) + + @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Add marker related options.""" - cli.add_command(markers) - - additional_build_parameters = [ - click.Option( - ["--strict-markers"], - is_flag=True, - help="Raise errors for unknown markers.", - default=False, - ), - click.Option( - ["-m", "marker_expression"], - metavar="MARKER_EXPRESSION", - type=str, - help="Select tasks via marker expressions.", - ), - click.Option( - ["-k", "expression"], - metavar="EXPRESSION", - type=str, - help="Select tasks via expressions on task ids.", - ), - ] - for command in ("build", "clean", "collect"): - cli.commands[command].params.extend(additional_build_parameters) + settings_builders["build"].option_groups["markers"] = Markers() @hookimpl diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index c3e32f0b..65751b3d 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -8,12 +8,9 @@ from typing import Iterable import click +import typed_settings as ts from click import Context -from sqlalchemy.engine import URL -from sqlalchemy.engine import make_url -from sqlalchemy.exc import ArgumentError -from _pytask.config_utils import set_defaults_from_config from _pytask.path import import_path from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import register_hook_impls_from_modules @@ -22,96 +19,7 @@ if TYPE_CHECKING: from pluggy import PluginManager - -_CONFIG_OPTION = click.Option( - ["-c", "--config"], - callback=set_defaults_from_config, - is_eager=True, - expose_value=False, - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - allow_dash=False, - path_type=Path, - resolve_path=True, - ), - help="Path to configuration file.", -) -"""click.Option: An option for the --config flag.""" - - -_IGNORE_OPTION = click.Option( - ["--ignore"], - type=str, - multiple=True, - help=( - "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " - "for more info." - ), - default=[], -) -"""click.Option: An option for the --ignore flag.""" - - -_PATH_ARGUMENT = click.Argument( - ["paths"], - nargs=-1, - type=click.Path(exists=True, resolve_path=True, path_type=Path), - is_eager=True, -) -"""click.Argument: An argument for paths.""" - - -_VERBOSE_OPTION = click.Option( - ["-v", "--verbose"], - type=click.IntRange(0, 2), - default=1, - help="Make pytask verbose (>= 0) or quiet (= 0).", -) -"""click.Option: An option to control pytask's verbosity.""" - - -_EDITOR_URL_SCHEME_OPTION = click.Option( - ["--editor-url-scheme"], - default="file", - help=( - "Use file, vscode, pycharm or a custom url scheme to add URLs to task " - "ids to quickly jump to the task definition. Use no_link to disable URLs." - ), -) -"""click.Option: An option to embed URLs in task ids.""" - - -def _database_url_callback( - ctx: Context, # noqa: ARG001 - name: str, # noqa: ARG001 - value: str | None, -) -> URL | None: - """Check the url for the database.""" - # Since sqlalchemy v2.0.19, we need to shortcircuit here. - if value is None: - return None - - try: - return make_url(value) - except ArgumentError: - msg = ( - "The 'database_url' must conform to sqlalchemy's url standard: " - "https://docs.sqlalchemy.org/en/latest/core/engines.html#backend-specific-urls." - ) - raise click.BadParameter(msg) from None - - -_DATABASE_URL_OPTION = click.Option( - ["--database-url"], - type=str, - help="Url to the database.", - default=None, - show_default="sqlite:///.../.pytask/pytask.sqlite3", - callback=_database_url_callback, -) + from _pytask.settings import SettingsBuilder def _hook_module_callback( @@ -168,26 +76,80 @@ def pytask_add_hooks(pm: PluginManager) -> None: return parsed_modules -_HOOK_MODULE_OPTION = click.Option( - ["--hook-module"], - type=str, - help="Path to a Python module that contains hook implementations.", - multiple=True, - is_eager=True, - callback=_hook_module_callback, +def _path_callback( + ctx: Context, # noqa: ARG001 + param: click.Parameter, # noqa: ARG001 + value: tuple[Path, ...], +) -> tuple[Path, ...]: + """Convert paths to Path objects.""" + return value or (Path.cwd(),) + + +@ts.settings +class Common: + """Common settings for the command line interface.""" + + editor_url_scheme: str = ts.option( + default="file", + click={"param_decls": ["--editor-url-scheme"]}, + help=( + "Use file, vscode, pycharm or a custom url scheme to add URLs to task " + "ids to quickly jump to the task definition. Use no_link to disable URLs." + ), + ) + ignore: tuple[str, ...] = ts.option( + factory=tuple, + help=( + "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " + "for more info." + ), + click={"param_decls": ["--ignore"], "multiple": True}, + ) + verbose: int = ts.option( + default=1, + help="Make pytask verbose (>= 0) or quiet (= 0).", + click={ + "param_decls": ["-v", "--verbose"], + "type": click.IntRange(0, 2), + "count": True, + }, + ) + hook_module: tuple[str, ...] = ts.option( + factory=list, + help="Path to a Python module that contains hook implementations.", + click={ + "param_decls": ["--hook-module"], + "multiple": True, + "is_eager": True, + "callback": _hook_module_callback, + }, + ) + paths: tuple[Path, ...] = ts.option( + factory=tuple, + click={ + "param_decls": ["--paths"], + "type": click.Path(exists=True, resolve_path=True, path_type=Path), + "multiple": True, + "callback": _path_callback, + "hidden": True, + }, + ) + pm: PluginManager | None = ts.option(default=None, click={"hidden": True}) + + +_PATH_ARGUMENT = click.Argument( + ["paths"], + nargs=-1, + type=click.Path(exists=True, resolve_path=True, path_type=Path), ) +"""click.Argument: An argument for paths.""" @hookimpl(trylast=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Register general markers.""" - for command in ("build", "clean", "collect", "dag", "profile"): - cli.commands[command].params.extend((_DATABASE_URL_OPTION,)) - for command in ("build", "clean", "collect", "dag", "markers", "profile"): - cli.commands[command].params.extend( - (_CONFIG_OPTION, _HOOK_MODULE_OPTION, _PATH_ARGUMENT) - ) - for command in ("build", "clean", "collect", "profile"): - cli.commands[command].params.extend([_IGNORE_OPTION, _EDITOR_URL_SCHEME_OPTION]) - for command in ("build",): - cli.commands[command].params.append(_VERBOSE_OPTION) + for settings_builder in settings_builders.values(): + settings_builder.arguments.append(_PATH_ARGUMENT) + settings_builder.option_groups["common"] = Common() diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index 2a10d25b..f988a7ec 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -37,26 +37,26 @@ def pytask_add_hooks(pm: PluginManager) -> None: """Add hooks.""" builtin_hook_impl_modules = ( "_pytask.build", - # "_pytask.capture", + "_pytask.capture", # "_pytask.clean", - # "_pytask.collect", + "_pytask.collect", # "_pytask.collect_command", - # "_pytask.config", - # "_pytask.dag", + "_pytask.config", + "_pytask.dag", # "_pytask.dag_command", - # "_pytask.database", + "_pytask.database", "_pytask.debugging", - # "_pytask.execute", - # "_pytask.live", - # "_pytask.logging", - # "_pytask.mark", - # "_pytask.nodes", - # "_pytask.parameters", - # "_pytask.persist", + "_pytask.execute", + "_pytask.live", + "_pytask.logging", + "_pytask.mark", + "_pytask.nodes", + "_pytask.parameters", + "_pytask.persist", # "_pytask.profile", - # "_pytask.skipping", - # "_pytask.task", - # "_pytask.warnings", + "_pytask.skipping", + "_pytask.task", + "_pytask.warnings", ) register_hook_impls_from_modules(pm, builtin_hook_impl_modules) diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 71ec5158..09c62d20 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -13,6 +13,7 @@ from typing import Generator import click +import typed_settings as ts from rich.table import Table from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column @@ -33,6 +34,7 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session +from _pytask.settings import SettingsBuilder from _pytask.traceback import Traceback if TYPE_CHECKING: @@ -58,10 +60,22 @@ class Runtime(BaseTable): duration: Mapped[float] +@ts.settings +class Base: + export: _ExportFormats = ts.option( + default=_ExportFormats.NO, + help="Export the profile in the specified format.", + ) + + @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend the command line interface.""" - cli.add_command(profile) + settings_builders["profile"] = SettingsBuilder( + name="profile", function=profile, base_settings=Base + ) @hookimpl diff --git a/src/_pytask/settings.py b/src/_pytask/settings.py new file mode 100644 index 00000000..4142692a --- /dev/null +++ b/src/_pytask/settings.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING +from typing import Any +from typing import Callable +from typing import cast + +import attrs +import click +import typed_settings as ts +from attrs import define +from attrs import field +from typed_settings.cli_click import OptionGroupFactory +from typed_settings.exceptions import ConfigFileLoadError +from typed_settings.exceptions import ConfigFileNotFoundError +from typed_settings.types import OptionList +from typed_settings.types import SettingsClass +from typed_settings.types import SettingsDict + +from _pytask.click import ColoredCommand +from _pytask.console import console + +if TYPE_CHECKING: + from pathlib import Path + + from typed_settings.loaders import Loader + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib # type: ignore[no-redef] + + +__all__ = ["Settings", "SettingsBuilder", "TomlFormat"] + + +Settings = Any + + +@define +class SettingsBuilder: + name: str + function: Callable[..., Any] + base_settings: Any + option_groups: dict[str, Any] = field(factory=dict) + arguments: list[Any] = field(factory=list) + + def build_settings(self) -> Any: + return ts.combine("Settings", self.base_settings, self.option_groups) + + def build_command(self, loaders: list[Loader]) -> Any: + settings = self.build_settings() + command = ts.click_options( + settings, loaders, decorator_factory=OptionGroupFactory() + )(self.function) + command = click.command(name=self.name, cls=ColoredCommand)(command) + command.params.extend(self.arguments) + return command + + +class TomlFormat: + """ + Support for TOML files. Read settings from the given *section*. + + Args: + section: The config file section to load settings from. + """ + + def __init__( + self, + section: str | None, + exclude: list[str] | None = None, + deprecated: str = "", + ) -> None: + self.section = section + self.exclude = exclude or [] + self.deprecated = deprecated + + def __call__( + self, + path: Path, + settings_cls: SettingsClass, # noqa: ARG002 + options: OptionList, # noqa: ARG002 + ) -> SettingsDict: + """ + Load settings from a TOML file and return them as a dict. + + Args: + path: The path to the config file. + options: The list of available settings. + settings_cls: The base settings class for all options. If ``None``, load + top level settings. + + Return: + A dict with the loaded settings. + + Raise: + ConfigFileNotFoundError: If *path* does not exist. + ConfigFileLoadError: If *path* cannot be read/loaded/decoded. + """ + try: + with path.open("rb") as f: + settings = tomllib.load(f) + except FileNotFoundError as e: + raise ConfigFileNotFoundError(str(e)) from e + except (PermissionError, tomllib.TOMLDecodeError) as e: + raise ConfigFileLoadError(str(e)) from e + if self.section is not None: + sections = self.section.split(".") + for s in sections: + try: + settings = settings[s] + except KeyError: # noqa: PERF203 + return {} + for key in self.exclude: + settings.pop(key, None) + + if self.deprecated: + console.print(self.deprecated) + return cast(SettingsDict, settings) + + +def load_settings(settings_cls: Any) -> Any: + """Load the settings.""" + loaders = create_settings_loaders() + return ts.load_settings(settings_cls, loaders) + + +def create_settings_loaders() -> list[Loader]: + """Create the loaders for the settings.""" + return [ + ts.FileLoader( + files=[ts.find("pyproject.toml")], + env_var=None, + formats={ + "*.toml": TomlFormat( + section="tool.pytask.ini_options", + deprecated=( + "[skipped]Deprecation Warning! Configuring pytask in the " + r"section \[tool.pytask.ini_options] is deprecated. " + r"Please, use \[tool.pytask] instead." + "[/]\n\n" + ), + ) + }, + ), + ts.FileLoader( + files=[ts.find("pyproject.toml")], + env_var=None, + formats={ + "*.toml": TomlFormat(section="tool.pytask", exclude=["ini_options"]) + }, + ), + ts.EnvLoader(prefix="PYTASK_", nested_delimiter="_"), + ] + + +def update_settings(settings: Any, updates: dict[str, Any]) -> Any: + """Update the settings recursively with some updates.""" + names = [i for i in dir(settings) if not i.startswith("_")] + for name in names: + if name in updates: + value = updates[name] + if value in ((), []): + continue + + setattr(settings, name, updates[name]) + + if attrs.has(getattr(settings, name)): + update_settings(getattr(settings, name), updates) + + return settings diff --git a/src/_pytask/warnings.py b/src/_pytask/warnings.py index b5986c84..288e5d4b 100644 --- a/src/_pytask/warnings.py +++ b/src/_pytask/warnings.py @@ -7,7 +7,7 @@ from typing import Any from typing import Generator -import click +import typed_settings as ts from attrs import define from rich.padding import Padding from rich.panel import Panel @@ -25,21 +25,31 @@ from _pytask.node_protocols import PTask from _pytask.session import Session + from _pytask.settings import SettingsBuilder + + +@ts.settings +class Warnings: + """Settings for warnings.""" + + filterwarnings: list[str] = ts.option( + factory=list, + click={"param_decls": ["--filterwarnings"]}, + help="Add a filter for a warning to a task.", + ) + disable_warnings: bool = ts.option( + default=False, + click={"param_decls": ["--disable-warnings"], "is_flag": True}, + help="Disables the summary for warnings.", + ) @hookimpl -def pytask_extend_command_line_interface(cli: click.Group) -> None: +def pytask_extend_command_line_interface( + settings_builders: dict[str, SettingsBuilder], +) -> None: """Extend the cli.""" - cli.commands["build"].params.extend( - [ - click.Option( - ["--disable-warnings"], - is_flag=True, - default=False, - help="Disables the summary for warnings.", - ) - ] - ) + settings_builders["build"].option_groups["warnings"] = Warnings() @hookimpl diff --git a/tests/test_capture.py b/tests/test_capture.py index f947c783..eed4c53a 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -569,9 +569,8 @@ def test_simple_resume_suspend(self): pytest.raises(AssertionError, cap.suspend) assert repr(cap) == ( - "".format( # noqa: UP032 - cap.targetfd_save, cap.tmpfile - ) + f"" ) # Should not crash with missing "_old". assert repr(cap.syscapture) == ( diff --git a/tests/test_jupyter/test_functional_interface.ipynb b/tests/test_jupyter/test_functional_interface.ipynb index 3abd995c..c74eb5e2 100644 --- a/tests/test_jupyter/test_functional_interface.ipynb +++ b/tests/test_jupyter/test_functional_interface.ipynb @@ -9,10 +9,11 @@ "source": [ "from pathlib import Path\n", "\n", - "from typing_extensions import Annotated\n", - "\n", "import pytask\n", - "from pytask import ExitCode, PathNode, PythonNode" + "from pytask import ExitCode\n", + "from pytask import PathNode\n", + "from pytask import PythonNode\n", + "from typing_extensions import Annotated" ] }, { diff --git a/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb index fe0125df..5226c122 100644 --- a/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb +++ b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb @@ -9,10 +9,11 @@ "source": [ "from pathlib import Path\n", "\n", - "from typing_extensions import Annotated\n", - "\n", "import pytask\n", - "from pytask import ExitCode, PathNode, PythonNode" + "from pytask import ExitCode\n", + "from pytask import PathNode\n", + "from pytask import PythonNode\n", + "from typing_extensions import Annotated" ] }, { diff --git a/tests/test_jupyter/test_task_generator.ipynb b/tests/test_jupyter/test_task_generator.ipynb index 2ef6aa61..d6d4c4b8 100644 --- a/tests/test_jupyter/test_task_generator.ipynb +++ b/tests/test_jupyter/test_task_generator.ipynb @@ -11,10 +11,11 @@ "\n", "from pathlib import Path\n", "\n", - "from typing_extensions import Annotated\n", - "\n", "import pytask\n", - "from pytask import DirectoryNode, ExitCode, task" + "from pytask import DirectoryNode\n", + "from pytask import ExitCode\n", + "from pytask import task\n", + "from typing_extensions import Annotated" ] }, { @@ -32,7 +33,7 @@ "\n", "@task(after=task_create_files, is_generator=True)\n", "def task_generator_copy_files(\n", - " paths: Annotated[list[Path], DirectoryNode(pattern=\"[ab].txt\")]\n", + " paths: Annotated[list[Path], DirectoryNode(pattern=\"[ab].txt\")],\n", "):\n", " for path in paths:\n", "\n", From 639b60360559f981b51a4c89a0cb3dde991c0e08 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 23 Mar 2024 11:55:56 +0100 Subject: [PATCH 03/15] FIx. --- src/_pytask/build.py | 58 +----- src/_pytask/capture.py | 7 +- src/_pytask/clean.py | 53 +---- src/_pytask/cli.py | 4 +- src/_pytask/collect_command.py | 2 +- src/_pytask/config.py | 15 +- src/_pytask/dag_command.py | 2 +- src/_pytask/database.py | 48 +---- src/_pytask/debugging.py | 5 +- src/_pytask/execute.py | 3 +- src/_pytask/hookspecs.py | 9 +- src/_pytask/live.py | 5 +- src/_pytask/logging.py | 8 +- src/_pytask/mark/__init__.py | 42 +--- src/_pytask/parameters.py | 5 +- src/_pytask/persist.py | 4 +- src/_pytask/profile.py | 26 +-- src/_pytask/settings.py | 340 +++++++++++++++++---------------- src/_pytask/settings_utils.py | 170 +++++++++++++++++ src/_pytask/skipping.py | 4 +- src/_pytask/task.py | 3 +- src/_pytask/warnings.py | 8 +- 22 files changed, 414 insertions(+), 407 deletions(-) create mode 100644 src/_pytask/settings_utils.py diff --git a/src/_pytask/build.py b/src/_pytask/build.py index bb703422..a2240dc8 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -29,9 +29,9 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings import SettingsBuilder -from _pytask.settings import create_settings_loaders -from _pytask.settings import update_settings +from _pytask.settings_utils import SettingsBuilder +from _pytask.settings_utils import create_settings_loaders +from _pytask.settings_utils import update_settings from _pytask.traceback import remove_internal_traceback_frames_from_exc_info if TYPE_CHECKING: @@ -39,49 +39,7 @@ from typing import NoReturn from _pytask.node_protocols import PTask - - -@ts.settings -class Base: - debug_pytask: bool = ts.option( - default=False, - click={"param_decls": ("--debug-pytask",), "is_flag": True}, - help="Trace all function calls in the plugin framework.", - ) - stop_after_first_failure: bool = ts.option( - default=False, - click={"param_decls": ("-x", "--stop-after-first-failure"), "is_flag": True}, - help="Stop after the first failure.", - ) - max_failures: float = ts.option( - default=float("inf"), - click={"param_decls": ("--max-failures",)}, - help="Stop after some failures.", - ) - show_errors_immediately: bool = ts.option( - default=False, - click={"param_decls": ("--show-errors-immediately",), "is_flag": True}, - help="Show errors with tracebacks as soon as the task fails.", - ) - show_traceback: bool = ts.option( - default=True, - click={"param_decls": ("--show-traceback", "--show-no-traceback")}, - help="Choose whether tracebacks should be displayed or not.", - ) - dry_run: bool = ts.option( - default=False, - click={"param_decls": ("--dry-run",), "is_flag": True}, - help="Perform a dry-run.", - ) - force: bool = ts.option( - default=False, - click={"param_decls": ("-f", "--force"), "is_flag": True}, - help="Execute a task even if it succeeded successfully before.", - ) - check_casing_of_paths: bool = ts.option( - default=True, - click={"param_decls": ("--check-casing-of-paths",), "hidden": True}, - ) + from _pytask.settings import Settings @hookimpl(tryfirst=True) @@ -89,15 +47,11 @@ def pytask_extend_command_line_interface( settings_builders: dict[str, SettingsBuilder], ) -> None: """Extend the command line interface.""" - settings_builders["build"] = SettingsBuilder( - name="build", - function=build_command, - base_settings=Base, - ) + settings_builders["build"] = SettingsBuilder(name="build", function=build_command) @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Fill cache of file hashes with stored hashes.""" with suppress(Exception): path = config["root"] / ".pytask" / "file_hashes.json" diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index a3286d36..f53f5592 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -50,7 +50,8 @@ if TYPE_CHECKING: from _pytask.node_protocols import PTask - from _pytask.settings import SettingsBuilder + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder @ts.settings @@ -83,7 +84,7 @@ def pytask_extend_command_line_interface( @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse configuration. Note that, ``-s`` is a shortcut for ``--capture=no``. @@ -96,7 +97,7 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Initialize the CaptureManager.""" pluginmanager = config["pm"] capman = CaptureManager(config["capture"]) diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index ea357025..ae3416d3 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -2,7 +2,6 @@ from __future__ import annotations -import enum import itertools import shutil import sys @@ -13,10 +12,8 @@ from typing import Iterable import click -import typed_settings as ts from attrs import define -from _pytask.click import EnumChoice from _pytask.console import console from _pytask.exceptions import CollectionError from _pytask.git import get_all_files @@ -31,7 +28,10 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings import SettingsBuilder +from _pytask.settings import Clean +from _pytask.settings import Settings +from _pytask.settings import _CleanMode +from _pytask.settings_utils import SettingsBuilder from _pytask.shared import to_list from _pytask.traceback import Traceback from _pytask.tree_util import tree_leaves @@ -40,62 +40,21 @@ from typing import NoReturn -class _CleanMode(enum.Enum): - DRY_RUN = "dry-run" - FORCE = "force" - INTERACTIVE = "interactive" - - _DEFAULT_EXCLUDE: list[str] = [".git/*"] -_HELP_TEXT_MODE = ( - "Choose 'dry-run' to print the paths of files/directories which would be removed, " - "'interactive' for a confirmation prompt for every path, and 'force' to remove all " - "unknown paths at once." -) - - -@ts.settings -class Base: - directories: bool = ts.option( - default=False, - help="Remove whole directories.", - click={"is_flag": True, "param_decls": ["-d", "--directories"]}, - ) - exclude: tuple[str, ...] = ts.option( - factory=tuple, - help="A filename pattern to exclude files from the cleaning process.", - click={ - "multiple": True, - "metavar": "PATTERN", - "param_decls": ["-e", "--exclude"], - }, - ) - mode: _CleanMode = ts.option( - default=_CleanMode.DRY_RUN, - help=_HELP_TEXT_MODE, - click={"type": EnumChoice(_CleanMode), "param_decls": ["-m", "--mode"]}, - ) - quiet: bool = ts.option( - default=False, - help="Do not print the names of the removed paths.", - click={"is_flag": True, "param_decls": ["-q", "--quiet"]}, - ) - - @hookimpl(tryfirst=True) def pytask_extend_command_line_interface( settings_builders: dict[str, SettingsBuilder], ) -> None: """Extend the command line interface.""" settings_builders["clean"] = SettingsBuilder( - name="clean", function=clean, base_settings=Base + name="clean", function=clean, base_settings=Clean ) @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" config["exclude"] = to_list(config["exclude"]) + _DEFAULT_EXCLUDE diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index 87534e80..63bbada9 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -9,8 +9,8 @@ from _pytask.click import ColoredGroup from _pytask.pluginmanager import storage -from _pytask.settings import SettingsBuilder -from _pytask.settings import create_settings_loaders +from _pytask.settings_utils import SettingsBuilder +from _pytask.settings_utils import create_settings_loaders _CONTEXT_SETTINGS: dict[str, Any] = { "help_option_names": ("-h", "--help"), diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index f0527894..5f3d0a9a 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -33,7 +33,7 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings import SettingsBuilder +from _pytask.settings_utils import SettingsBuilder from _pytask.tree_util import tree_leaves if TYPE_CHECKING: diff --git a/src/_pytask/config.py b/src/_pytask/config.py index 86423ac3..c6a9ee59 100644 --- a/src/_pytask/config.py +++ b/src/_pytask/config.py @@ -5,16 +5,16 @@ import tempfile from pathlib import Path from typing import TYPE_CHECKING -from typing import Any from _pytask.pluginmanager import hookimpl -from _pytask.shared import parse_markers from _pytask.shared import parse_paths from _pytask.shared import to_list if TYPE_CHECKING: from pluggy import PluginManager + from _pytask.settings import Settings + _IGNORED_FOLDERS: list[str] = [".git/*", ".venv/*"] @@ -61,20 +61,15 @@ def is_file_system_case_sensitive() -> bool: @hookimpl -def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, Any]: +def pytask_configure(pm: PluginManager, config: Settings) -> Settings: """Configure pytask.""" - # Add all values by default so that many plugins do not need to copy over values. - config = {"pm": pm, "markers": {}, **raw_config} - config["markers"] = parse_markers(config["markers"]) - pm.hook.pytask_parse_config(config=config) pm.hook.pytask_post_parse(config=config) - return config @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" config["root"].joinpath(".pytask").mkdir(exist_ok=True, parents=True) @@ -112,6 +107,6 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Sort markers alphabetically.""" config["markers"] = {k: config["markers"][k] for k in sorted(config["markers"])} diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index 4cc161d5..370e383f 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -28,7 +28,7 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings import SettingsBuilder +from _pytask.settings_utils import SettingsBuilder from _pytask.shared import parse_paths from _pytask.shared import reduce_names_of_multiple_nodes from _pytask.shared import to_list diff --git a/src/_pytask/database.py b/src/_pytask/database.py index d00beb6e..9af433ae 100644 --- a/src/_pytask/database.py +++ b/src/_pytask/database.py @@ -4,54 +4,16 @@ from pathlib import Path from typing import TYPE_CHECKING -from typing import Any -import typed_settings as ts -from click import BadParameter -from click import Context -from sqlalchemy.engine import URL from sqlalchemy.engine import make_url -from sqlalchemy.exc import ArgumentError from _pytask.database_utils import create_database from _pytask.pluginmanager import hookimpl +from _pytask.settings import Database if TYPE_CHECKING: - from _pytask.settings import SettingsBuilder - - -def _database_url_callback( - ctx: Context, # noqa: ARG001 - name: str, # noqa: ARG001 - value: str | None, -) -> URL | None: - """Check the url for the database.""" - # Since sqlalchemy v2.0.19, we need to shortcircuit here. - if value is None: - return None - - try: - return make_url(value) - except ArgumentError: - msg = ( - "The 'database_url' must conform to sqlalchemy's url standard: " - "https://docs.sqlalchemy.org/en/latest/core/engines.html#backend-specific-urls." - ) - raise BadParameter(msg) from None - - -@ts.settings -class Database: - """Settings for the database.""" - - database_url: str = ts.option( - default=None, - help="Url to the database.", - click={ - "show_default": "sqlite:///.../.pytask/pytask.sqlite3", - "callback": _database_url_callback, - }, - ) + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder @hookimpl @@ -64,7 +26,7 @@ def pytask_extend_command_line_interface( @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" # Set default. if not config["database_url"]: @@ -92,6 +54,6 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Post-parse the configuration.""" create_database(config["database_url"]) diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index 4f959a03..95605639 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -28,7 +28,8 @@ from _pytask.capture import CaptureManager from _pytask.live import LiveManager from _pytask.session import Session - from _pytask.settings import SettingsBuilder + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder def _pdbcls_callback( @@ -84,7 +85,7 @@ def pytask_extend_command_line_interface( @hookimpl(trylast=True) -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Post parse the configuration. Register the plugins in this step to let other plugins influence the pdb or trace diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index 7ef1dd00..07533dd6 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -47,10 +47,11 @@ if TYPE_CHECKING: from _pytask.session import Session + from _pytask.settings import Settings @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Adjust the configuration after intermediate values have been parsed.""" if config["show_errors_immediately"]: config["pm"].register(ShowErrorsImmediatelyPlugin) diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 3ec8100b..2a616eca 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -26,7 +26,8 @@ from _pytask.reports import CollectionReport from _pytask.reports import ExecutionReport from _pytask.session import Session - from _pytask.settings import SettingsBuilder + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder hookspec = pluggy.HookspecMarker("pytask") @@ -69,7 +70,7 @@ def pytask_extend_command_line_interface( @hookspec(firstresult=True) -def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, Any]: +def pytask_configure(pm: PluginManager, config: Settings) -> Settings: """Configure pytask. The main hook implementation which controls the configuration and calls subordinated @@ -79,12 +80,12 @@ def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, @hookspec -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse configuration that is from CLI or file.""" @hookspec -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Post parsing. This hook allows to consolidate the configuration in case some plugins might be diff --git a/src/_pytask/live.py b/src/_pytask/live.py index 0ab53880..9df66bbb 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -28,7 +28,8 @@ from _pytask.reports import CollectionReport from _pytask.reports import ExecutionReport from _pytask.session import Session - from _pytask.settings import SettingsBuilder + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder @ts.settings @@ -57,7 +58,7 @@ def pytask_extend_command_line_interface( @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Post-parse the configuration.""" live_manager = LiveManager() config["pm"].register(live_manager, "live_manager") diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index 7e814cdf..f1e5c420 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -7,7 +7,6 @@ import sys import warnings from typing import TYPE_CHECKING -from typing import Any from typing import NamedTuple import pluggy @@ -27,7 +26,8 @@ from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.session import Session - from _pytask.settings import SettingsBuilder + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder with contextlib.suppress(ImportError): @@ -65,7 +65,7 @@ def pytask_extend_command_line_interface( @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse configuration.""" if config["editor_url_scheme"] not in ("no_link", "file") and IS_WINDOWS_TERMINAL: config["editor_url_scheme"] = "file" @@ -78,7 +78,7 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: # Set class variables on traceback object. Traceback.show_locals = config["show_locals"] # Set class variables on Executionreport. diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 44b642d0..9deed001 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -7,12 +7,9 @@ from typing import AbstractSet from typing import Any -import click -import typed_settings as ts from attrs import define from rich.table import Table -from _pytask.click import ColoredCommand from _pytask.console import console from _pytask.dag_utils import task_and_preceding_tasks from _pytask.exceptions import ConfigurationError @@ -26,6 +23,9 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session +from _pytask.settings import Markers +from _pytask.settings import Settings +from _pytask.settings_utils import SettingsBuilder from _pytask.shared import parse_markers if TYPE_CHECKING: @@ -34,7 +34,6 @@ import networkx as nx from _pytask.node_protocols import PTask - from _pytask.settings import SettingsBuilder __all__ = [ @@ -51,7 +50,6 @@ ] -@click.command(cls=ColoredCommand) def markers(**raw_config: Any) -> NoReturn: """Show all registered markers.""" raw_config["command"] = "markers" @@ -77,47 +75,25 @@ def markers(**raw_config: Any) -> NoReturn: sys.exit(session.exit_code) -@ts.settings -class Markers: - """Settings for markers.""" - - strict_markers: bool = ts.option( - default=False, - click={"param_decls": ["--strict-markers"], "is_flag": True}, - help="Raise errors for unknown markers.", - ) - marker_expression: str = ts.option( - default="", - click={ - "param_decls": ["-m", "marker_expression"], - "metavar": "MARKER_EXPRESSION", - }, - help="Select tasks via marker expressions.", - ) - expression: str = ts.option( - default="", - click={"param_decls": ["-k", "expression"], "metavar": "EXPRESSION"}, - help="Select tasks via expressions on task ids.", - ) - - @hookimpl def pytask_extend_command_line_interface( settings_builders: dict[str, SettingsBuilder], ) -> None: """Add marker related options.""" - settings_builders["build"].option_groups["markers"] = Markers() + settings_builders["markers"] = SettingsBuilder(name="markers", function=markers) + for settings_builder in settings_builders.values(): + settings_builder.option_groups["markers"] = Markers() @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse marker related options.""" MARK_GEN.config = config @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: - config["markers"] = parse_markers(config["markers"]) +def pytask_post_parse(config: Settings) -> None: + config.markers.markers = parse_markers(config["markers"]) @define(slots=True) diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index 65751b3d..e0a021b5 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -10,6 +10,7 @@ import click import typed_settings as ts from click import Context +from pluggy import PluginManager # noqa: TCH002 from _pytask.path import import_path from _pytask.pluginmanager import hookimpl @@ -17,9 +18,7 @@ from _pytask.pluginmanager import storage if TYPE_CHECKING: - from pluggy import PluginManager - - from _pytask.settings import SettingsBuilder + from _pytask.settings_utils import SettingsBuilder def _hook_module_callback( diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 7cb272f0..9a54807e 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -3,7 +3,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -from typing import Any from _pytask.dag_utils import node_and_neighbors from _pytask.database_utils import has_node_changed @@ -18,10 +17,11 @@ from _pytask.node_protocols import PTask from _pytask.reports import ExecutionReport from _pytask.session import Session + from _pytask.settings import Settings @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Add the marker to the configuration.""" config["markers"]["persist"] = ( "Prevent execution of a task if all products exist and even if something has " diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 09c62d20..513b8028 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -3,7 +3,6 @@ from __future__ import annotations import csv -import enum import json import sys import time @@ -13,7 +12,6 @@ from typing import Generator import click -import typed_settings as ts from rich.table import Table from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column @@ -34,7 +32,8 @@ from _pytask.pluginmanager import hookimpl from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings import SettingsBuilder +from _pytask.settings import _ExportFormats +from _pytask.settings_utils import SettingsBuilder from _pytask.traceback import Traceback if TYPE_CHECKING: @@ -42,12 +41,7 @@ from typing import NoReturn from _pytask.reports import ExecutionReport - - -class _ExportFormats(enum.Enum): - NO = "no" - JSON = "json" - CSV = "csv" + from _pytask.settings import Settings class Runtime(BaseTable): @@ -60,26 +54,16 @@ class Runtime(BaseTable): duration: Mapped[float] -@ts.settings -class Base: - export: _ExportFormats = ts.option( - default=_ExportFormats.NO, - help="Export the profile in the specified format.", - ) - - @hookimpl(tryfirst=True) def pytask_extend_command_line_interface( settings_builders: dict[str, SettingsBuilder], ) -> None: """Extend the command line interface.""" - settings_builders["profile"] = SettingsBuilder( - name="profile", function=profile, base_settings=Base - ) + settings_builders["profile"] = SettingsBuilder(name="profile", function=profile) @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Register the export option.""" config["pm"].register(ExportNameSpace) config["pm"].register(DurationNameSpace) diff --git a/src/_pytask/settings.py b/src/_pytask/settings.py index 4142692a..a7270321 100644 --- a/src/_pytask/settings.py +++ b/src/_pytask/settings.py @@ -1,173 +1,175 @@ from __future__ import annotations -import sys -from typing import TYPE_CHECKING -from typing import Any -from typing import Callable -from typing import cast - -import attrs -import click +from enum import Enum + import typed_settings as ts -from attrs import define -from attrs import field -from typed_settings.cli_click import OptionGroupFactory -from typed_settings.exceptions import ConfigFileLoadError -from typed_settings.exceptions import ConfigFileNotFoundError -from typed_settings.types import OptionList -from typed_settings.types import SettingsClass -from typed_settings.types import SettingsDict - -from _pytask.click import ColoredCommand -from _pytask.console import console - -if TYPE_CHECKING: - from pathlib import Path - - from typed_settings.loaders import Loader - -if sys.version_info >= (3, 11): - import tomllib -else: - import tomli as tomllib # type: ignore[no-redef] - - -__all__ = ["Settings", "SettingsBuilder", "TomlFormat"] - - -Settings = Any - - -@define -class SettingsBuilder: - name: str - function: Callable[..., Any] - base_settings: Any - option_groups: dict[str, Any] = field(factory=dict) - arguments: list[Any] = field(factory=list) - - def build_settings(self) -> Any: - return ts.combine("Settings", self.base_settings, self.option_groups) - - def build_command(self, loaders: list[Loader]) -> Any: - settings = self.build_settings() - command = ts.click_options( - settings, loaders, decorator_factory=OptionGroupFactory() - )(self.function) - command = click.command(name=self.name, cls=ColoredCommand)(command) - command.params.extend(self.arguments) - return command - - -class TomlFormat: - """ - Support for TOML files. Read settings from the given *section*. - - Args: - section: The config file section to load settings from. - """ - - def __init__( - self, - section: str | None, - exclude: list[str] | None = None, - deprecated: str = "", - ) -> None: - self.section = section - self.exclude = exclude or [] - self.deprecated = deprecated - - def __call__( - self, - path: Path, - settings_cls: SettingsClass, # noqa: ARG002 - options: OptionList, # noqa: ARG002 - ) -> SettingsDict: - """ - Load settings from a TOML file and return them as a dict. - - Args: - path: The path to the config file. - options: The list of available settings. - settings_cls: The base settings class for all options. If ``None``, load - top level settings. - - Return: - A dict with the loaded settings. - - Raise: - ConfigFileNotFoundError: If *path* does not exist. - ConfigFileLoadError: If *path* cannot be read/loaded/decoded. - """ - try: - with path.open("rb") as f: - settings = tomllib.load(f) - except FileNotFoundError as e: - raise ConfigFileNotFoundError(str(e)) from e - except (PermissionError, tomllib.TOMLDecodeError) as e: - raise ConfigFileLoadError(str(e)) from e - if self.section is not None: - sections = self.section.split(".") - for s in sections: - try: - settings = settings[s] - except KeyError: # noqa: PERF203 - return {} - for key in self.exclude: - settings.pop(key, None) - - if self.deprecated: - console.print(self.deprecated) - return cast(SettingsDict, settings) - - -def load_settings(settings_cls: Any) -> Any: - """Load the settings.""" - loaders = create_settings_loaders() - return ts.load_settings(settings_cls, loaders) - - -def create_settings_loaders() -> list[Loader]: - """Create the loaders for the settings.""" - return [ - ts.FileLoader( - files=[ts.find("pyproject.toml")], - env_var=None, - formats={ - "*.toml": TomlFormat( - section="tool.pytask.ini_options", - deprecated=( - "[skipped]Deprecation Warning! Configuring pytask in the " - r"section \[tool.pytask.ini_options] is deprecated. " - r"Please, use \[tool.pytask] instead." - "[/]\n\n" - ), - ) - }, - ), - ts.FileLoader( - files=[ts.find("pyproject.toml")], - env_var=None, - formats={ - "*.toml": TomlFormat(section="tool.pytask", exclude=["ini_options"]) - }, +from click import BadParameter +from click import Context +from sqlalchemy.engine import URL +from sqlalchemy.engine import make_url +from sqlalchemy.exc import ArgumentError + +from _pytask.click import EnumChoice + + +@ts.settings +class Build: + stop_after_first_failure: bool = ts.option( + default=False, + click={"param_decls": ("-x", "--stop-after-first-failure"), "is_flag": True}, + help="Stop after the first failure.", + ) + max_failures: float = ts.option( + default=float("inf"), + click={"param_decls": ("--max-failures",)}, + help="Stop after some failures.", + ) + show_errors_immediately: bool = ts.option( + default=False, + click={"param_decls": ("--show-errors-immediately",), "is_flag": True}, + help="Show errors with tracebacks as soon as the task fails.", + ) + show_traceback: bool = ts.option( + default=True, + click={"param_decls": ("--show-traceback", "--show-no-traceback")}, + help="Choose whether tracebacks should be displayed or not.", + ) + dry_run: bool = ts.option( + default=False, + click={"param_decls": ("--dry-run",), "is_flag": True}, + help="Perform a dry-run.", + ) + force: bool = ts.option( + default=False, + click={"param_decls": ("-f", "--force"), "is_flag": True}, + help="Execute a task even if it succeeded successfully before.", + ) + check_casing_of_paths: bool = ts.option( + default=True, + click={"param_decls": ("--check-casing-of-paths",), "hidden": True}, + ) + + +class _ExportFormats(Enum): + NO = "no" + JSON = "json" + CSV = "csv" + + +@ts.settings +class Profile: + export: _ExportFormats = ts.option( + default=_ExportFormats.NO, + help="Export the profile in the specified format.", + ) + + +class _CleanMode(Enum): + DRY_RUN = "dry-run" + FORCE = "force" + INTERACTIVE = "interactive" + + +@ts.settings +class Clean: + directories: bool = ts.option( + default=False, + help="Remove whole directories.", + click={"is_flag": True, "param_decls": ["-d", "--directories"]}, + ) + exclude: tuple[str, ...] = ts.option( + factory=tuple, + help="A filename pattern to exclude files from the cleaning process.", + click={ + "multiple": True, + "metavar": "PATTERN", + "param_decls": ["-e", "--exclude"], + }, + ) + mode: _CleanMode = ts.option( + default=_CleanMode.DRY_RUN, + help=( + "Choose 'dry-run' to print the paths of files/directories which would be " + "removed, 'interactive' for a confirmation prompt for every path, and " + "'force' to remove all unknown paths at once." ), - ts.EnvLoader(prefix="PYTASK_", nested_delimiter="_"), - ] - - -def update_settings(settings: Any, updates: dict[str, Any]) -> Any: - """Update the settings recursively with some updates.""" - names = [i for i in dir(settings) if not i.startswith("_")] - for name in names: - if name in updates: - value = updates[name] - if value in ((), []): - continue - - setattr(settings, name, updates[name]) - - if attrs.has(getattr(settings, name)): - update_settings(getattr(settings, name), updates) - - return settings + click={"type": EnumChoice(_CleanMode), "param_decls": ["-m", "--mode"]}, + ) + quiet: bool = ts.option( + default=False, + help="Do not print the names of the removed paths.", + click={"is_flag": True, "param_decls": ["-q", "--quiet"]}, + ) + + +def _database_url_callback( + ctx: Context, # noqa: ARG001 + name: str, # noqa: ARG001 + value: str | None, +) -> URL | None: + """Check the url for the database.""" + # Since sqlalchemy v2.0.19, we need to shortcircuit here. + if value is None: + return None + + try: + return make_url(value) + except ArgumentError: + msg = ( + "The 'database_url' must conform to sqlalchemy's url standard: " + "https://docs.sqlalchemy.org/en/latest/core/engines.html#backend-specific-urls." + ) + raise BadParameter(msg) from None + + +@ts.settings +class Database: + """Settings for the database.""" + + database_url: str = ts.option( + default=None, + help="Url to the database.", + click={ + "show_default": "sqlite:///.../.pytask/pytask.sqlite3", + "callback": _database_url_callback, + }, + ) + + +@ts.settings +class Markers: + """Settings for markers.""" + + strict_markers: bool = ts.option( + default=False, + click={"param_decls": ["--strict-markers"], "is_flag": True}, + help="Raise errors for unknown markers.", + ) + marker_expression: str = ts.option( + default="", + click={ + "param_decls": ["-m", "marker_expression"], + "metavar": "MARKER_EXPRESSION", + }, + help="Select tasks via marker expressions.", + ) + expression: str = ts.option( + default="", + click={"param_decls": ["-k", "expression"], "metavar": "EXPRESSION"}, + help="Select tasks via expressions on task ids.", + ) + + +@ts.settings +class Settings: + debug_pytask: bool = ts.option( + default=False, + click={"param_decls": ("--debug-pytask",), "is_flag": True}, + help="Trace all function calls in the plugin framework.", + ) + + # markers: Markers + # profile: Profile + # build: Build + # clean: Clean + # database: Database diff --git a/src/_pytask/settings_utils.py b/src/_pytask/settings_utils.py new file mode 100644 index 00000000..d701bfb9 --- /dev/null +++ b/src/_pytask/settings_utils.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING +from typing import Any +from typing import Callable +from typing import cast + +import attrs +import click +import typed_settings as ts +from attrs import define +from attrs import field +from typed_settings.cli_click import OptionGroupFactory +from typed_settings.exceptions import ConfigFileLoadError +from typed_settings.exceptions import ConfigFileNotFoundError +from typed_settings.types import OptionList +from typed_settings.types import SettingsClass +from typed_settings.types import SettingsDict + +from _pytask.click import ColoredCommand +from _pytask.console import console +from _pytask.settings import Settings + +if TYPE_CHECKING: + from pathlib import Path + + from typed_settings.loaders import Loader + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib # type: ignore[no-redef] + + +__all__ = ["SettingsBuilder", "TomlFormat"] + + +@define +class SettingsBuilder: + name: str + function: Callable[..., Any] + option_groups: dict[str, Any] = field(factory=dict) + arguments: list[Any] = field(factory=list) + + def build_settings(self) -> Any: + return ts.combine("Settings", Settings, self.option_groups) + + def build_command(self, loaders: list[Loader]) -> Any: + settings = self.build_settings() + command = ts.click_options( + settings, loaders, decorator_factory=OptionGroupFactory() + )(self.function) + command = click.command(name=self.name, cls=ColoredCommand)(command) + command.params.extend(self.arguments) + return command + + +class TomlFormat: + """ + Support for TOML files. Read settings from the given *section*. + + Args: + section: The config file section to load settings from. + """ + + def __init__( + self, + section: str | None, + exclude: list[str] | None = None, + deprecated: str = "", + ) -> None: + self.section = section + self.exclude = exclude or [] + self.deprecated = deprecated + + def __call__( + self, + path: Path, + settings_cls: SettingsClass, # noqa: ARG002 + options: OptionList, # noqa: ARG002 + ) -> SettingsDict: + """ + Load settings from a TOML file and return them as a dict. + + Args: + path: The path to the config file. + options: The list of available settings. + settings_cls: The base settings class for all options. If ``None``, load + top level settings. + + Return: + A dict with the loaded settings. + + Raise: + ConfigFileNotFoundError: If *path* does not exist. + ConfigFileLoadError: If *path* cannot be read/loaded/decoded. + """ + try: + with path.open("rb") as f: + settings = tomllib.load(f) + except FileNotFoundError as e: + raise ConfigFileNotFoundError(str(e)) from e + except (PermissionError, tomllib.TOMLDecodeError) as e: + raise ConfigFileLoadError(str(e)) from e + if self.section is not None: + sections = self.section.split(".") + for s in sections: + try: + settings = settings[s] + except KeyError: # noqa: PERF203 + return {} + for key in self.exclude: + settings.pop(key, None) + + if self.deprecated: + console.print(self.deprecated) + return cast(SettingsDict, settings) + + +def load_settings(settings_cls: Any) -> Any: + """Load the settings.""" + loaders = create_settings_loaders() + return ts.load_settings(settings_cls, loaders) + + +def create_settings_loaders() -> list[Loader]: + """Create the loaders for the settings.""" + return [ + ts.FileLoader( + files=[ts.find("pyproject.toml")], + env_var=None, + formats={ + "*.toml": TomlFormat( + section="tool.pytask.ini_options", + deprecated=( + "[skipped]Deprecation Warning! Configuring pytask in the " + r"section \[tool.pytask.ini_options] is deprecated. " + r"Please, use \[tool.pytask] instead." + "[/]\n\n" + ), + ) + }, + ), + ts.FileLoader( + files=[ts.find("pyproject.toml")], + env_var=None, + formats={ + "*.toml": TomlFormat(section="tool.pytask", exclude=["ini_options"]) + }, + ), + ts.EnvLoader(prefix="PYTASK_", nested_delimiter="_"), + ] + + +def update_settings(settings: Any, updates: dict[str, Any]) -> Any: + """Update the settings recursively with some updates.""" + names = [i for i in dir(settings) if not i.startswith("_")] + for name in names: + if name in updates: + value = updates[name] + if value in ((), []): + continue + + setattr(settings, name, updates[name]) + + if attrs.has(getattr(settings, name)): + update_settings(getattr(settings, name), updates) + + return settings diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index a7678154..49bc8951 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -3,7 +3,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -from typing import Any from _pytask.dag_utils import descending_tasks from _pytask.mark import Mark @@ -20,6 +19,7 @@ from _pytask.node_protocols import PTask from _pytask.reports import ExecutionReport from _pytask.session import Session + from _pytask.settings import Settings def skip_ancestor_failed(reason: str = "No reason provided.") -> str: @@ -33,7 +33,7 @@ def skipif(condition: bool, *, reason: str) -> tuple[bool, str]: @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" markers = { "skip": "Skip a task and all its dependent tasks.", diff --git a/src/_pytask/task.py b/src/_pytask/task.py index 902f6eb0..ec1c0c36 100644 --- a/src/_pytask/task.py +++ b/src/_pytask/task.py @@ -17,10 +17,11 @@ from _pytask.reports import CollectionReport from _pytask.session import Session + from _pytask.settings import Settings @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" config["markers"]["task"] = ( "Mark a function as a task regardless of its name. Or mark tasks which are " diff --git a/src/_pytask/warnings.py b/src/_pytask/warnings.py index 288e5d4b..6f6b5e51 100644 --- a/src/_pytask/warnings.py +++ b/src/_pytask/warnings.py @@ -4,7 +4,6 @@ from collections import defaultdict from typing import TYPE_CHECKING -from typing import Any from typing import Generator import typed_settings as ts @@ -25,7 +24,8 @@ from _pytask.node_protocols import PTask from _pytask.session import Session - from _pytask.settings import SettingsBuilder + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder @ts.settings @@ -53,14 +53,14 @@ def pytask_extend_command_line_interface( @hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: +def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" config["filterwarnings"] = parse_filterwarnings(config.get("filterwarnings")) config["markers"]["filterwarnings"] = "Add a filter for a warning to a task." @hookimpl -def pytask_post_parse(config: dict[str, Any]) -> None: +def pytask_post_parse(config: Settings) -> None: """Activate the warnings plugin if not disabled.""" if not config["disable_warnings"]: config["pm"].register(WarningsNameSpace) From 986f364171c1b4df6ffd6193505656108e919b6a Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 24 Mar 2024 16:00:13 +0100 Subject: [PATCH 04/15] temp commit. --- pyproject.toml | 1 + src/_pytask/capture.py | 24 +------ src/_pytask/collect_command.py | 4 +- src/_pytask/dag_command.py | 4 +- src/_pytask/database.py | 4 +- src/_pytask/debugging.py | 2 +- src/_pytask/logging.py | 4 +- src/_pytask/parameters.py | 98 +------------------------- src/_pytask/persist.py | 2 +- src/_pytask/settings.py | 123 +++++++++++++++++++++++++++++++++ src/_pytask/settings_utils.py | 1 + src/_pytask/skipping.py | 2 +- src/_pytask/task.py | 2 +- src/_pytask/task_utils.py | 1 - src/_pytask/warnings.py | 6 +- tests/test_collect_utils.py | 2 +- 16 files changed, 141 insertions(+), 139 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 55a8b460..92032323 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,7 @@ ignore = [ "S603", # Call check with subprocess.run. "S607", # Call subprocess.run with partial executable path. "SLF001", # access private members. + "TCH002", ] diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index f53f5592..2943144a 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -41,11 +41,10 @@ from typing import TextIO from typing import final -import typed_settings as ts - from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture from _pytask.pluginmanager import hookimpl +from _pytask.settings import Capture from _pytask.shared import convert_to_enum if TYPE_CHECKING: @@ -54,27 +53,6 @@ from _pytask.settings_utils import SettingsBuilder -@ts.settings -class Capture: - """Settings for capturing.""" - - capture: CaptureMethod = ts.option( - default=CaptureMethod.FD, - click={"param_decls": ["--capture"]}, - help="Per task capturing method.", - ) - s: bool = ts.option( - default=False, - click={"param_decls": ["-s"], "is_flag": True}, - help="Shortcut for --capture=no.", - ) - show_capture: ShowCapture = ts.option( - default=ShowCapture.ALL, - click={"param_decls": ["--show-capture"]}, - help="Choose which captured output should be shown for failed tasks.", - ) - - @hookimpl def pytask_extend_command_line_interface( settings_builders: dict[str, SettingsBuilder], diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 5f3d0a9a..54c93cef 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -55,9 +55,7 @@ def pytask_extend_command_line_interface( settings_builders: dict[str, SettingsBuilder], ) -> None: """Extend the command line interface.""" - settings_builders["collect"] = SettingsBuilder( - name="collect", function=collect, base_settings=Base - ) + settings_builders["collect"] = SettingsBuilder(name="collect", function=collect) def collect(**raw_config: Any | None) -> NoReturn: diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index 54b0d8c2..096dad3a 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -73,9 +73,7 @@ def pytask_extend_command_line_interface( settings_builders: dict[str, SettingsBuilder], ) -> None: """Extend the command line interface.""" - settings_builders["dag"] = SettingsBuilder( - name="dag", function=dag, base_settings=Base - ) + settings_builders["dag"] = SettingsBuilder(name="dag", function=dag) def dag(**raw_config: Any) -> int: diff --git a/src/_pytask/database.py b/src/_pytask/database.py index 9af433ae..3b185b26 100644 --- a/src/_pytask/database.py +++ b/src/_pytask/database.py @@ -29,8 +29,8 @@ def pytask_extend_command_line_interface( def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" # Set default. - if not config["database_url"]: - config["database_url"] = make_url( + if not config.database.database_url: + config.database.database_url = make_url( f"sqlite:///{config['root'].joinpath('.pytask').as_posix()}/pytask.sqlite3" ) diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index 95605639..5a2fc2a8 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -121,7 +121,7 @@ class PytaskPDB: """Pseudo PDB that defers to the real pdb.""" _pluginmanager: PluginManager | None = None - _config: dict[str, Any] | None = None + _config: Settings | None = None _saved: ClassVar[list[tuple[Any, ...]]] = [] _recursive_debug: int = 0 _wrapped_pdb_cls: tuple[type[pdb.Pdb], type[pdb.Pdb]] | None = None diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index f1e5c420..e0e242b8 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -67,8 +67,8 @@ def pytask_extend_command_line_interface( @hookimpl def pytask_parse_config(config: Settings) -> None: """Parse configuration.""" - if config["editor_url_scheme"] not in ("no_link", "file") and IS_WINDOWS_TERMINAL: - config["editor_url_scheme"] = "file" + if config.editor_url_scheme not in ("no_link", "file") and IS_WINDOWS_TERMINAL: + config.editor_url_scheme = "file" warnings.warn( "Windows Terminal does not support url schemes to applications, yet." "See https://github.com/pytask-dev/pytask/issues/171 for more information. " diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index e0a021b5..7a1e1f34 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -2,77 +2,18 @@ from __future__ import annotations -import importlib.util from pathlib import Path from typing import TYPE_CHECKING -from typing import Iterable import click import typed_settings as ts from click import Context -from pluggy import PluginManager # noqa: TCH002 +from pluggy import PluginManager -from _pytask.path import import_path from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import register_hook_impls_from_modules -from _pytask.pluginmanager import storage if TYPE_CHECKING: - from _pytask.settings_utils import SettingsBuilder - - -def _hook_module_callback( - ctx: Context, - name: str, # noqa: ARG001 - value: tuple[str, ...], -) -> Iterable[str | Path]: - """Register the user's hook modules from the configuration file.""" - if not value: - return value - - parsed_modules = [] - for module_name in value: - if module_name.endswith(".py"): - path = Path(module_name) - if ctx.params["config"]: - path = ctx.params["config"].parent.joinpath(path).resolve() - else: - path = Path.cwd().joinpath(path).resolve() - - if not path.exists(): - msg = ( - f"The hook module {path} does not exist. " - "Please provide a valid path." - ) - raise click.BadParameter(msg) - module = import_path(path, ctx.params["root"]) - parsed_modules.append(module.__name__) - else: - spec = importlib.util.find_spec(module_name) - if spec is None: - msg = ( - f"The hook module {module_name!r} is not importable. " - "Please provide a valid module name." - ) - raise click.BadParameter(msg) - parsed_modules.append(module_name) - - # If there are hook modules, we register a hook implementation to add them. - # ``pytask_add_hooks`` is a historic hook specification, so even command line - # options can be added. - if parsed_modules: - - class HookModule: - @staticmethod - @hookimpl - def pytask_add_hooks(pm: PluginManager) -> None: - """Add hooks.""" - register_hook_impls_from_modules(pm, parsed_modules) - - pm = storage.get() - pm.register(HookModule) - - return parsed_modules + from _pytask.settings import SettingsBuilder def _path_callback( @@ -88,41 +29,6 @@ def _path_callback( class Common: """Common settings for the command line interface.""" - editor_url_scheme: str = ts.option( - default="file", - click={"param_decls": ["--editor-url-scheme"]}, - help=( - "Use file, vscode, pycharm or a custom url scheme to add URLs to task " - "ids to quickly jump to the task definition. Use no_link to disable URLs." - ), - ) - ignore: tuple[str, ...] = ts.option( - factory=tuple, - help=( - "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " - "for more info." - ), - click={"param_decls": ["--ignore"], "multiple": True}, - ) - verbose: int = ts.option( - default=1, - help="Make pytask verbose (>= 0) or quiet (= 0).", - click={ - "param_decls": ["-v", "--verbose"], - "type": click.IntRange(0, 2), - "count": True, - }, - ) - hook_module: tuple[str, ...] = ts.option( - factory=list, - help="Path to a Python module that contains hook implementations.", - click={ - "param_decls": ["--hook-module"], - "multiple": True, - "is_eager": True, - "callback": _hook_module_callback, - }, - ) paths: tuple[Path, ...] = ts.option( factory=tuple, click={ diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 9a54807e..86e578a4 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -23,7 +23,7 @@ @hookimpl def pytask_parse_config(config: Settings) -> None: """Add the marker to the configuration.""" - config["markers"]["persist"] = ( + config.markers.markers["persist"] = ( "Prevent execution of a task if all products exist and even if something has " "changed (dependencies, source file, products). This decorator might be useful " "for expensive tasks where only the formatting of the file has changed. The " diff --git a/src/_pytask/settings.py b/src/_pytask/settings.py index a6a22611..b611c37a 100644 --- a/src/_pytask/settings.py +++ b/src/_pytask/settings.py @@ -1,15 +1,80 @@ from __future__ import annotations +import importlib.util from enum import Enum +from pathlib import Path +from typing import Iterable +import click import typed_settings as ts from click import BadParameter from click import Context +from pluggy import PluginManager from sqlalchemy.engine import URL from sqlalchemy.engine import make_url from sqlalchemy.exc import ArgumentError +from _pytask.capture_utils import CaptureMethod +from _pytask.capture_utils import ShowCapture from _pytask.click import EnumChoice +from _pytask.path import import_path +from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import register_hook_impls_from_modules +from _pytask.pluginmanager import storage + + +def _hook_module_callback( + ctx: Context, + name: str, # noqa: ARG001 + value: tuple[str, ...], +) -> Iterable[str | Path]: + """Register the user's hook modules from the configuration file.""" + if not value: + return value + + parsed_modules = [] + for module_name in value: + if module_name.endswith(".py"): + path = Path(module_name) + if ctx.params["config"]: + path = ctx.params["config"].parent.joinpath(path).resolve() + else: + path = Path.cwd().joinpath(path).resolve() + + if not path.exists(): + msg = ( + f"The hook module {path} does not exist. " + "Please provide a valid path." + ) + raise click.BadParameter(msg) + module = import_path(path, ctx.params["root"]) + parsed_modules.append(module.__name__) + else: + spec = importlib.util.find_spec(module_name) + if spec is None: + msg = ( + f"The hook module {module_name!r} is not importable. " + "Please provide a valid module name." + ) + raise click.BadParameter(msg) + parsed_modules.append(module_name) + + # If there are hook modules, we register a hook implementation to add them. + # ``pytask_add_hooks`` is a historic hook specification, so even command line + # options can be added. + if parsed_modules: + + class HookModule: + @staticmethod + @hookimpl + def pytask_add_hooks(pm: PluginManager) -> None: + """Add hooks.""" + register_hook_impls_from_modules(pm, parsed_modules) + + pm = storage.get() + pm.register(HookModule) + + return parsed_modules @ts.settings @@ -145,6 +210,7 @@ class Markers: click={"param_decls": ["--strict-markers"], "is_flag": True}, help="Raise errors for unknown markers.", ) + markers: dict[str, str] = ts.option(factory=dict, click={"hidden": True}) marker_expression: str = ts.option( default="", click={ @@ -167,3 +233,60 @@ class Settings: click={"param_decls": ("--debug-pytask",), "is_flag": True}, help="Trace all function calls in the plugin framework.", ) + editor_url_scheme: str = ts.option( + default="file", + click={"param_decls": ["--editor-url-scheme"]}, + help=( + "Use file, vscode, pycharm or a custom url scheme to add URLs to task " + "ids to quickly jump to the task definition. Use no_link to disable URLs." + ), + ) + ignore: tuple[str, ...] = ts.option( + factory=tuple, + help=( + "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " + "for more info." + ), + click={"param_decls": ["--ignore"], "multiple": True}, + ) + verbose: int = ts.option( + default=1, + help="Make pytask verbose (>= 0) or quiet (= 0).", + click={ + "param_decls": ["-v", "--verbose"], + "type": click.IntRange(0, 2), + "count": True, + }, + ) + hook_module: tuple[str, ...] = ts.option( + factory=list, + help="Path to a Python module that contains hook implementations.", + click={ + "param_decls": ["--hook-module"], + "multiple": True, + "is_eager": True, + "callback": _hook_module_callback, + }, + ) + config_file: Path | None = None + + +@ts.settings +class Capture: + """Settings for capturing.""" + + capture: CaptureMethod = ts.option( + default=CaptureMethod.FD, + click={"param_decls": ["--capture"]}, + help="Per task capturing method.", + ) + s: bool = ts.option( + default=False, + click={"param_decls": ["-s"], "is_flag": True}, + help="Shortcut for --capture=no.", + ) + show_capture: ShowCapture = ts.option( + default=ShowCapture.ALL, + click={"param_decls": ["--show-capture"]}, + help="Choose which captured output should be shown for failed tasks.", + ) diff --git a/src/_pytask/settings_utils.py b/src/_pytask/settings_utils.py index e9a5fd1f..5d6d93b5 100644 --- a/src/_pytask/settings_utils.py +++ b/src/_pytask/settings_utils.py @@ -120,6 +120,7 @@ def __call__( if self.deprecated and not _ALREADY_PRINTED_DEPRECATION_MSG: _ALREADY_PRINTED_DEPRECATION_MSG = True console.print(self.deprecated) + settings["config_file"] = path return cast(SettingsDict, settings) diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index 49bc8951..bcf9e0bb 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -43,7 +43,7 @@ def pytask_parse_config(config: Settings) -> None: "executed and have not been changed.", "skipif": "Skip a task and all its dependent tasks if a condition is met.", } - config["markers"] = {**config["markers"], **markers} + config.markers.markers = {**config.markers.markers, **markers} @hookimpl diff --git a/src/_pytask/task.py b/src/_pytask/task.py index ec1c0c36..343ad237 100644 --- a/src/_pytask/task.py +++ b/src/_pytask/task.py @@ -23,7 +23,7 @@ @hookimpl def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" - config["markers"]["task"] = ( + config.markers.markers["task"] = ( "Mark a function as a task regardless of its name. Or mark tasks which are " "repeated in a loop. See this tutorial for more information: " "[link https://bit.ly/3DWrXS3]https://bit.ly/3DWrXS3[/]." diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py index a9f556c4..c1bba26e 100644 --- a/src/_pytask/task_utils.py +++ b/src/_pytask/task_utils.py @@ -21,7 +21,6 @@ if TYPE_CHECKING: from pathlib import Path - __all__ = [ "COLLECTED_TASKS", "parse_collected_tasks_with_task_marker", diff --git a/src/_pytask/warnings.py b/src/_pytask/warnings.py index 6f6b5e51..59883a0d 100644 --- a/src/_pytask/warnings.py +++ b/src/_pytask/warnings.py @@ -15,7 +15,6 @@ from _pytask.pluginmanager import hookimpl from _pytask.warnings_utils import WarningReport from _pytask.warnings_utils import catch_warnings_for_item -from _pytask.warnings_utils import parse_filterwarnings if TYPE_CHECKING: from rich.console import Console @@ -34,7 +33,7 @@ class Warnings: filterwarnings: list[str] = ts.option( factory=list, - click={"param_decls": ["--filterwarnings"]}, + click={"hidden": True}, help="Add a filter for a warning to a task.", ) disable_warnings: bool = ts.option( @@ -55,8 +54,7 @@ def pytask_extend_command_line_interface( @hookimpl def pytask_parse_config(config: Settings) -> None: """Parse the configuration.""" - config["filterwarnings"] = parse_filterwarnings(config.get("filterwarnings")) - config["markers"]["filterwarnings"] = "Add a filter for a warning to a task." + config.markers.markers["filterwarnings"] = "Add a filter for a warning to a task." @hookimpl diff --git a/tests/test_collect_utils.py b/tests/test_collect_utils.py index aabe7036..b23674ac 100644 --- a/tests/test_collect_utils.py +++ b/tests/test_collect_utils.py @@ -2,7 +2,7 @@ import pytest from _pytask.collect_utils import _find_args_with_product_annotation -from pytask import Product # noqa: TCH002 +from pytask import Product from typing_extensions import Annotated From d51d043ec353c9f0e556ad07ea4557284aba7c42 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 24 Mar 2024 16:02:37 +0100 Subject: [PATCH 05/15] fix. --- docs/source/changes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/changes.md b/docs/source/changes.md index 3f7ce9b6..b4440088 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -26,6 +26,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`579` fixes an interaction with `--pdb` and `--trace` and task that return. The debugging modes swallowed the return and `None` was returned. Closes {issue}`574`. - {pull}`581` simplifies the code for tracebacks and unpublishes some utility functions. +- {pull}`582` use typed-settings to parse configuration files and create the CLI. ## 0.4.6 - 2024-03-13 From c76d127300cafc3ff97bf948118f3fe63bc7d43e Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 10 Apr 2024 19:11:27 +0200 Subject: [PATCH 06/15] fix. --- src/_pytask/capture.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index 71a1af83..a9e476fd 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -43,7 +43,6 @@ from typing import TextIO from typing import final -import click from typing_extensions import Self from _pytask.capture_utils import CaptureMethod From c643eb001b017d2ba679a3541fd6ae5de7adf35e Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 10 Apr 2024 20:22:03 +0200 Subject: [PATCH 07/15] fix issue with base class. --- .gitignore | 3 + .pre-commit-config.yaml | 2 +- pyproject.toml | 31 +++++++--- src/_pytask/logging.py | 7 ++- src/_pytask/parameters.py | 102 ++++++++++++++++++++++++++++++++ src/_pytask/settings.py | 106 +--------------------------------- src/_pytask/settings_utils.py | 21 ++++++- src/_pytask/traceback.py | 2 +- 8 files changed, 156 insertions(+), 118 deletions(-) diff --git a/.gitignore b/.gitignore index d5e3ea27..50159a20 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ src/_pytask/_version.py *.pkl tests/test_jupyter/*.txt +.mypy_cache +.pytest_cache +.ruff_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7bbbf60..7e92c15f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -109,7 +109,7 @@ repos: hooks: - id: check-manifest args: [--no-build-isolation] - additional_dependencies: [setuptools-scm, toml, wheel] + additional_dependencies: [hatchling, hatch-vcs] - repo: meta hooks: - id: check-hooks-apply diff --git a/pyproject.toml b/pyproject.toml index e17a38f3..983b61cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,3 @@ -[build-system] -build-backend = "setuptools.build_meta" -requires = ["setuptools>=64", "setuptools_scm>=8"] - -[tool.setuptools_scm] -version_file = "src/_pytask/_version.py" - [project] name = "pytask" description = "In its highest aspirations, pytask tries to be pytest as a build system." @@ -38,6 +31,7 @@ dependencies = [ "rich", "sqlalchemy>=2", 'tomli>=1; python_version < "3.11"', + "typed-settings[option-groups]", 'typing-extensions; python_version < "3.9"', "universal-pathlib>=0.2.2", ] @@ -90,6 +84,26 @@ Tracker = "https://github.com/pytask-dev/pytask/issues" [project.scripts] pytask = "pytask:cli" +[build-system] +requires = ["hatchling", "hatch_vcs"] +build-backend = "hatchling.build" + +[tool.rye] +managed = true +dev-dependencies = [] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.hooks.vcs] +version-file = "src/_pytask/_version.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/pytask"] + +[tool.hatch.version] +source = "vcs" + [tool.setuptools] include-package-data = true zip-safe = false @@ -103,6 +117,9 @@ license-files = ["LICENSE"] where = ["src"] namespaces = false +[tool.setuptools_scm] +version_file = "src/_pytask/_version.py" + [tool.ruff] target-version = "py38" fix = true diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index b34f168c..c8d8c757 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -68,8 +68,11 @@ def pytask_extend_command_line_interface( @hookimpl def pytask_parse_config(config: Settings) -> None: """Parse configuration.""" - if config.editor_url_scheme not in ("no_link", "file") and IS_WINDOWS_TERMINAL: - config.editor_url_scheme = "file" + if ( + config.common.editor_url_scheme not in ("no_link", "file") + and IS_WINDOWS_TERMINAL + ): + config.common.editor_url_scheme = "file" warnings.warn( "Windows Terminal does not support url schemes to applications, yet." "See https://github.com/pytask-dev/pytask/issues/171 for more information. " diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index 7a1e1f34..c6d71033 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -2,20 +2,79 @@ from __future__ import annotations +import importlib.util from pathlib import Path from typing import TYPE_CHECKING +from typing import Iterable import click import typed_settings as ts from click import Context from pluggy import PluginManager +from _pytask.path import import_path from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import register_hook_impls_from_modules +from _pytask.pluginmanager import storage if TYPE_CHECKING: from _pytask.settings import SettingsBuilder +def _hook_module_callback( + ctx: Context, + name: str, # noqa: ARG001 + value: tuple[str, ...], +) -> Iterable[str | Path]: + """Register the user's hook modules from the configuration file.""" + if not value: + return value + + parsed_modules = [] + for module_name in value: + if module_name.endswith(".py"): + path = Path(module_name) + if ctx.params["config"]: + path = ctx.params["config"].parent.joinpath(path).resolve() + else: + path = Path.cwd().joinpath(path).resolve() + + if not path.exists(): + msg = ( + f"The hook module {path} does not exist. " + "Please provide a valid path." + ) + raise click.BadParameter(msg) + module = import_path(path, ctx.params["root"]) + parsed_modules.append(module.__name__) + else: + spec = importlib.util.find_spec(module_name) + if spec is None: + msg = ( + f"The hook module {module_name!r} is not importable. " + "Please provide a valid module name." + ) + raise click.BadParameter(msg) + parsed_modules.append(module_name) + + # If there are hook modules, we register a hook implementation to add them. + # ``pytask_add_hooks`` is a historic hook specification, so even command line + # options can be added. + if parsed_modules: + + class HookModule: + @staticmethod + @hookimpl + def pytask_add_hooks(pm: PluginManager) -> None: + """Add hooks.""" + register_hook_impls_from_modules(pm, parsed_modules) + + pm = storage.get() + pm.register(HookModule) + + return parsed_modules + + def _path_callback( ctx: Context, # noqa: ARG001 param: click.Parameter, # noqa: ARG001 @@ -29,6 +88,49 @@ def _path_callback( class Common: """Common settings for the command line interface.""" + debug_pytask: bool = ts.option( + default=False, + click={"param_decls": ("--debug-pytask",), "is_flag": True}, + help="Trace all function calls in the plugin framework.", + ) + editor_url_scheme: str = ts.option( + default="file", + click={"param_decls": ["--editor-url-scheme"]}, + help=( + "Use file, vscode, pycharm or a custom url scheme to add URLs to task " + "ids to quickly jump to the task definition. Use no_link to disable URLs." + ), + ) + ignore: tuple[str, ...] = ts.option( + factory=tuple, + help=( + "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " + "for more info." + ), + click={"param_decls": ["--ignore"], "multiple": True}, + ) + verbose: int = ts.option( + default=1, + help="Make pytask verbose (>= 0) or quiet (= 0).", + click={ + "param_decls": ["-v", "--verbose"], + "type": click.IntRange(0, 2), + "count": True, + }, + ) + hook_module: tuple[str, ...] = ts.option( + factory=list, + help="Path to a Python module that contains hook implementations.", + click={ + "param_decls": ["--hook-module"], + "multiple": True, + "is_eager": True, + "callback": _hook_module_callback, + }, + ) + config_file: Path | None = ts.option( + default=None, click={"param_decls": ["--config-file"], "hidden": True} + ) paths: tuple[Path, ...] = ts.option( factory=tuple, click={ diff --git a/src/_pytask/settings.py b/src/_pytask/settings.py index b611c37a..94e09f1c 100644 --- a/src/_pytask/settings.py +++ b/src/_pytask/settings.py @@ -1,15 +1,10 @@ from __future__ import annotations -import importlib.util from enum import Enum -from pathlib import Path -from typing import Iterable -import click import typed_settings as ts from click import BadParameter from click import Context -from pluggy import PluginManager from sqlalchemy.engine import URL from sqlalchemy.engine import make_url from sqlalchemy.exc import ArgumentError @@ -17,64 +12,6 @@ from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture from _pytask.click import EnumChoice -from _pytask.path import import_path -from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import register_hook_impls_from_modules -from _pytask.pluginmanager import storage - - -def _hook_module_callback( - ctx: Context, - name: str, # noqa: ARG001 - value: tuple[str, ...], -) -> Iterable[str | Path]: - """Register the user's hook modules from the configuration file.""" - if not value: - return value - - parsed_modules = [] - for module_name in value: - if module_name.endswith(".py"): - path = Path(module_name) - if ctx.params["config"]: - path = ctx.params["config"].parent.joinpath(path).resolve() - else: - path = Path.cwd().joinpath(path).resolve() - - if not path.exists(): - msg = ( - f"The hook module {path} does not exist. " - "Please provide a valid path." - ) - raise click.BadParameter(msg) - module = import_path(path, ctx.params["root"]) - parsed_modules.append(module.__name__) - else: - spec = importlib.util.find_spec(module_name) - if spec is None: - msg = ( - f"The hook module {module_name!r} is not importable. " - "Please provide a valid module name." - ) - raise click.BadParameter(msg) - parsed_modules.append(module_name) - - # If there are hook modules, we register a hook implementation to add them. - # ``pytask_add_hooks`` is a historic hook specification, so even command line - # options can be added. - if parsed_modules: - - class HookModule: - @staticmethod - @hookimpl - def pytask_add_hooks(pm: PluginManager) -> None: - """Add hooks.""" - register_hook_impls_from_modules(pm, parsed_modules) - - pm = storage.get() - pm.register(HookModule) - - return parsed_modules @ts.settings @@ -227,48 +164,7 @@ class Markers: @ts.settings -class Settings: - debug_pytask: bool = ts.option( - default=False, - click={"param_decls": ("--debug-pytask",), "is_flag": True}, - help="Trace all function calls in the plugin framework.", - ) - editor_url_scheme: str = ts.option( - default="file", - click={"param_decls": ["--editor-url-scheme"]}, - help=( - "Use file, vscode, pycharm or a custom url scheme to add URLs to task " - "ids to quickly jump to the task definition. Use no_link to disable URLs." - ), - ) - ignore: tuple[str, ...] = ts.option( - factory=tuple, - help=( - "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " - "for more info." - ), - click={"param_decls": ["--ignore"], "multiple": True}, - ) - verbose: int = ts.option( - default=1, - help="Make pytask verbose (>= 0) or quiet (= 0).", - click={ - "param_decls": ["-v", "--verbose"], - "type": click.IntRange(0, 2), - "count": True, - }, - ) - hook_module: tuple[str, ...] = ts.option( - factory=list, - help="Path to a Python module that contains hook implementations.", - click={ - "param_decls": ["--hook-module"], - "multiple": True, - "is_eager": True, - "callback": _hook_module_callback, - }, - ) - config_file: Path | None = None +class Settings: ... @ts.settings diff --git a/src/_pytask/settings_utils.py b/src/_pytask/settings_utils.py index 5d6d93b5..d29b52e6 100644 --- a/src/_pytask/settings_utils.py +++ b/src/_pytask/settings_utils.py @@ -81,7 +81,7 @@ def __call__( self, path: Path, settings_cls: SettingsClass, # noqa: ARG002 - options: OptionList, # noqa: ARG002 + options: OptionList, ) -> SettingsDict: """ Load settings from a TOML file and return them as a dict. @@ -120,9 +120,26 @@ def __call__( if self.deprecated and not _ALREADY_PRINTED_DEPRECATION_MSG: _ALREADY_PRINTED_DEPRECATION_MSG = True console.print(self.deprecated) - settings["config_file"] = path + settings["common.config_file"] = path + settings = self._rewrite_paths_of_options(settings, options) return cast(SettingsDict, settings) + def _rewrite_paths_of_options( + self, settings: SettingsDict, options: OptionList + ) -> SettingsDict: + """Rewrite paths of options in the settings.""" + option_paths = {option.path for option in options} + for name in list(settings): + if name in option_paths: + continue + + for option_path in option_paths: + if name in option_path: + settings[option_path] = settings.pop(name) + break + + return settings + def load_settings(settings_cls: Any) -> Any: """Load the settings.""" diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py index 00242916..41394020 100644 --- a/src/_pytask/traceback.py +++ b/src/_pytask/traceback.py @@ -51,7 +51,7 @@ class Traceback: _show_locals: ClassVar[bool] = False suppress: ClassVar[tuple[Path, ...]] = ( _PLUGGY_DIRECTORY, - _PYTASK_DIRECTORY, + # _PYTASK_DIRECTORY, TREE_UTIL_LIB_DIRECTORY, ) From 6576671175b3283390295825d71800966bf88b91 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 21 Apr 2024 11:55:55 +0200 Subject: [PATCH 08/15] One settings object for each command. --- docs/source/reference_guides/api.md | 1 - pyproject.toml | 6 +- src/_pytask/build.py | 107 ++++++++++++++------- src/_pytask/capture.py | 30 +++++- src/_pytask/clean.py | 65 ++++++++++--- src/_pytask/cli.py | 22 +++-- src/_pytask/click.py | 33 +------ src/_pytask/collect.py | 7 +- src/_pytask/collect_command.py | 25 ++--- src/_pytask/config_utils.py | 16 ---- src/_pytask/dag_command.py | 14 +-- src/_pytask/data_catalog.py | 7 +- src/_pytask/debugging.py | 19 ++-- src/_pytask/hookspecs.py | 6 +- src/_pytask/live.py | 6 +- src/_pytask/logging.py | 6 +- src/_pytask/mark/__init__.py | 52 +++++++--- src/_pytask/parameters.py | 26 ++--- src/_pytask/pluginmanager.py | 8 +- src/_pytask/profile.py | 48 +++++----- src/_pytask/session.py | 7 +- src/_pytask/settings.py | 144 +--------------------------- src/_pytask/settings.pyi | 50 ++-------- src/_pytask/settings_utils.py | 127 +++++++++++++++++------- src/_pytask/traceback.py | 4 +- src/_pytask/warnings.py | 6 +- src/pytask/__init__.py | 2 - tests/test_click.py | 39 -------- 28 files changed, 398 insertions(+), 485 deletions(-) diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md index 4ec74a8e..25d0664f 100644 --- a/docs/source/reference_guides/api.md +++ b/docs/source/reference_guides/api.md @@ -10,7 +10,6 @@ pytask offers the following functionalities. ```{eval-rst} .. autoclass:: pytask.ColoredCommand .. autoclass:: pytask.ColoredGroup -.. autoclass:: pytask.EnumChoice ``` ## Compatibility diff --git a/pyproject.toml b/pyproject.toml index b80485bc..9c1848a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "sqlalchemy>=2", 'tomli>=1; python_version < "3.11"', 'typing-extensions; python_version < "3.9"', - "typed-settings[option-groups]", + "typed-settings[cattrs,option-groups]", "universal-pathlib>=0.2.2", ] @@ -225,5 +225,5 @@ end_of_line = "keep" # [tool.pytask] # debug_pytask = false -[tool.pytask.ini_options] -debug_pytask = true +# [tool.pytask.ini_options] +# debug_pytask = true diff --git a/src/_pytask/build.py b/src/_pytask/build.py index b4d6c47c..781affc6 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -15,7 +15,6 @@ from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture -from _pytask.config_utils import consolidate_settings_and_arguments from _pytask.console import console from _pytask.dag import create_dag from _pytask.exceptions import CollectionError @@ -24,12 +23,9 @@ from _pytask.exceptions import ResolvingDependenciesError from _pytask.outcomes import ExitCode from _pytask.path import HashPathCache -from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session from _pytask.settings_utils import SettingsBuilder -from _pytask.settings_utils import create_settings_loaders from _pytask.settings_utils import update_settings from _pytask.traceback import Traceback @@ -41,12 +37,49 @@ from _pytask.settings import Settings +@ts.settings +class Build: + stop_after_first_failure: bool = ts.option( + default=False, + click={"param_decls": ("-x", "--stop-after-first-failure"), "is_flag": True}, + help="Stop after the first failure.", + ) + max_failures: float = ts.option( + default=float("inf"), + click={"param_decls": ("--max-failures",)}, + help="Stop after some failures.", + ) + show_errors_immediately: bool = ts.option( + default=False, + click={"param_decls": ("--show-errors-immediately",), "is_flag": True}, + help="Show errors with tracebacks as soon as the task fails.", + ) + show_traceback: bool = ts.option( + default=True, + click={"param_decls": ("--show-traceback", "--show-no-traceback")}, + help="Choose whether tracebacks should be displayed or not.", + ) + dry_run: bool = ts.option( + default=False, + click={"param_decls": ("--dry-run",), "is_flag": True}, + help="Perform a dry-run.", + ) + force: bool = ts.option( + default=False, + click={"param_decls": ("-f", "--force"), "is_flag": True}, + help="Execute a task even if it succeeded successfully before.", + ) + check_casing_of_paths: bool = ts.option( + default=True, + click={"param_decls": ("--check-casing-of-paths",), "hidden": True}, + ) + + @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - settings_builders["build"] = SettingsBuilder(name="build", function=build_command) + settings_builder.commands["build"] = build_command + settings_builder.option_groups["build"] = Build() @hookimpl @@ -86,7 +119,6 @@ def build( # noqa: PLR0913 pdb: bool = False, pdb_cls: str = "", s: bool = False, - settings: Settings | None = None, show_capture: Literal["no", "stdout", "stderr", "all"] | ShowCapture = ShowCapture.ALL, show_errors_immediately: bool = False, @@ -146,8 +178,6 @@ def build( # noqa: PLR0913 ``--pdbcls=IPython.terminal.debugger:TerminalPdb`` s Shortcut for ``capture="no"``. - settings - The settings object that contains the configuration. show_capture Choose which captured output should be shown for failed tasks. show_errors_immediately @@ -203,27 +233,46 @@ def build( # noqa: PLR0913 "sort_table": sort_table, "stop_after_first_failure": stop_after_first_failure, "strict_markers": strict_markers, + "tasks": tasks, "task_files": task_files, "trace": trace, "verbose": verbose, **kwargs, } - if settings is None: - from _pytask.cli import settings_builders + from _pytask.cli import settings_builder + + settings = settings_builder.load_settings(kwargs=updates) + except (ConfigurationError, Exception): + console.print(Traceback(sys.exc_info())) + session = Session(exit_code=ExitCode.CONFIGURATION_FAILED) + else: + session = _internal_build(settings=settings, tasks=tasks) + return session + + +def build_command(settings: Any, **arguments: Any) -> NoReturn: + """Collect tasks, execute them and report the results. + + The default command. pytask collects tasks from the given paths or the + current working directory, executes them and reports the results. - pm = get_plugin_manager() - storage.store(pm) + """ + settings = update_settings(settings, arguments) + session = _internal_build(settings=settings) + sys.exit(session.exit_code) - settings = ts.load_settings( - settings_builders["build"].build_settings(), create_settings_loaders() - ) - else: - pm = storage.get() - settings = update_settings(settings, updates) - config_ = pm.hook.pytask_configure(pm=pm, config=settings) - session = Session.from_config(config_) +def _internal_build( + settings: Settings, + tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (), +) -> Session: + """Run pytask internally.""" + try: + config = settings.common.pm.hook.pytask_configure( + pm=settings.common.pm, config=settings + ) + session = Session(config=config, hook=config.common.pm.hook) session.attrs["tasks"] = tasks except (ConfigurationError, Exception): @@ -252,15 +301,3 @@ def build( # noqa: PLR0913 session.hook.pytask_unconfigure(session=session) return session - - -def build_command(settings: Any, **arguments: Any) -> NoReturn: - """Collect tasks, execute them and report the results. - - The default command. pytask collects tasks from the given paths or the - current working directory, executes them and reports the results. - - """ - settings = consolidate_settings_and_arguments(settings, arguments) - session = build(settings=settings) - sys.exit(session.exit_code) diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index 3d9d5621..b55680cb 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -43,11 +43,12 @@ from typing import TextIO from typing import final +import typed_settings as ts from typing_extensions import Self from _pytask.capture_utils import CaptureMethod +from _pytask.capture_utils import ShowCapture from _pytask.pluginmanager import hookimpl -from _pytask.settings import Capture if TYPE_CHECKING: from types import TracebackType @@ -57,12 +58,31 @@ from _pytask.settings_utils import SettingsBuilder +@ts.settings +class Capture: + """Settings for capturing.""" + + capture: CaptureMethod = ts.option( + default=CaptureMethod.FD, + click={"param_decls": ["--capture"]}, + help="Per task capturing method.", + ) + s: bool = ts.option( + default=False, + click={"param_decls": ["-s"], "is_flag": True}, + help="Shortcut for --capture=no.", + ) + show_capture: ShowCapture = ts.option( + default=ShowCapture.ALL, + click={"param_decls": ["--show-capture"]}, + help="Choose which captured output should be shown for failed tasks.", + ) + + @hookimpl -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Add CLI options for capturing output.""" - settings_builders["build"].option_groups["capture"] = Capture() + settings_builder.option_groups["capture"] = Capture() @hookimpl diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index 5eac052d..f3a6bbfe 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -5,12 +5,14 @@ import itertools import shutil import sys +from enum import Enum from typing import TYPE_CHECKING from typing import Any from typing import Generator from typing import Iterable import click +import typed_settings as ts from attrs import define from _pytask.console import console @@ -25,11 +27,8 @@ from _pytask.path import find_common_ancestor from _pytask.path import relative_to from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings import Settings -from _pytask.settings import _CleanMode -from _pytask.settings_utils import SettingsBuilder +from _pytask.settings_utils import update_settings from _pytask.traceback import Traceback from _pytask.tree_util import tree_leaves @@ -37,16 +36,56 @@ from pathlib import Path from typing import NoReturn + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder + _DEFAULT_EXCLUDE: tuple[str, ...] = (".git/*",) +class _CleanMode(Enum): + DRY_RUN = "dry-run" + FORCE = "force" + INTERACTIVE = "interactive" + + +@ts.settings +class Clean: + directories: bool = ts.option( + default=False, + help="Remove whole directories.", + click={"is_flag": True, "param_decls": ["-d", "--directories"]}, + ) + exclude: tuple[str, ...] = ts.option( + factory=tuple, + help="A filename pattern to exclude files from the cleaning process.", + click={ + "multiple": True, + "metavar": "PATTERN", + "param_decls": ["-e", "--exclude"], + }, + ) + mode: _CleanMode = ts.option( + default=_CleanMode.DRY_RUN, + help=( + "Choose 'dry-run' to print the paths of files/directories which would be " + "removed, 'interactive' for a confirmation prompt for every path, and " + "'force' to remove all unknown paths at once." + ), + click={"param_decls": ["-m", "--mode"]}, + ) + quiet: bool = ts.option( + default=False, + help="Do not print the names of the removed paths.", + click={"is_flag": True, "param_decls": ["-q", "--quiet"]}, + ) + + @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - settings_builders["clean"] = SettingsBuilder(name="clean", function=clean_command) + settings_builder.commands["clean"] = clean_command + settings_builder.option_groups["clean"] = Clean() @hookimpl @@ -55,15 +94,15 @@ def pytask_parse_config(config: Settings) -> None: config.clean.exclude = config.clean.exclude + _DEFAULT_EXCLUDE -def clean_command(**raw_config: Any) -> NoReturn: # noqa: C901, PLR0912 +def clean_command(settings: Settings, **arguments: Any) -> NoReturn: # noqa: C901, PLR0912 """Clean the provided paths by removing files unknown to pytask.""" - pm = storage.get() - raw_config["command"] = "clean" + settings = update_settings(settings, arguments) + pm = settings.common.pm try: # Duplication of the same mechanism in :func:`pytask.build`. - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + config = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session(config=config, hook=config.common.pm.hook) except Exception: # noqa: BLE001 # pragma: no cover session = Session(exit_code=ExitCode.CONFIGURATION_FAILED) diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index 63bbada9..38ec2ee7 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -7,10 +7,10 @@ import click from packaging.version import parse as parse_version +from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup from _pytask.pluginmanager import storage from _pytask.settings_utils import SettingsBuilder -from _pytask.settings_utils import create_settings_loaders _CONTEXT_SETTINGS: dict[str, Any] = { "help_option_names": ("-h", "--help"), @@ -24,14 +24,18 @@ _VERSION_OPTION_KWARGS = {} -def _extend_command_line_interface() -> dict[str, SettingsBuilder]: +def _extend_command_line_interface() -> SettingsBuilder: """Add parameters from plugins to the commandline interface.""" pm = storage.create() - settings_builders: dict[str, SettingsBuilder] = {} + settings_builder = SettingsBuilder() pm.hook.pytask_extend_command_line_interface.call_historic( - kwargs={"settings_builders": settings_builders} + kwargs={"settings_builder": settings_builder} ) - return settings_builders + return settings_builder + + +settings_builder = _extend_command_line_interface() +decorator = settings_builder.build_decorator() @click.group( @@ -45,9 +49,7 @@ def cli() -> None: """Manage your tasks with pytask.""" -settings_builders = _extend_command_line_interface() -settings_loaders = create_settings_loaders() - -for settings_builder in settings_builders.values(): - command = settings_builder.build_command(settings_loaders) +for name, func in settings_builder.commands.items(): + command = click.command(name=name, cls=ColoredCommand)(decorator(func)) + command.params.extend(settings_builder.arguments) cli.add_command(command) diff --git a/src/_pytask/click.py b/src/_pytask/click.py index 83ffaa1e..36d24d20 100644 --- a/src/_pytask/click.py +++ b/src/_pytask/click.py @@ -11,7 +11,6 @@ from typing import ClassVar import click -from click import Choice from click import Command from click import Context from click import Parameter @@ -29,37 +28,7 @@ from collections.abc import Sequence -__all__ = ["ColoredCommand", "ColoredGroup", "EnumChoice"] - - -class EnumChoice(Choice): - """An enum-based choice type. - - The implementation is copied from https://github.com/pallets/click/pull/2210 and - related discussion can be found in https://github.com/pallets/click/issues/605. - - In contrast to using :class:`click.Choice`, using this type ensures that the error - message does not show the enum members. - - In contrast to the proposed implementation in the PR, this implementation does not - use the members than rather the values of the enum. - - """ - - def __init__(self, enum_type: type[Enum], case_sensitive: bool = True) -> None: - super().__init__( - choices=[element.value for element in enum_type], - case_sensitive=case_sensitive, - ) - self.enum_type = enum_type - - def convert(self, value: Any, param: Parameter | None, ctx: Context | None) -> Any: - if isinstance(value, Enum): - value = value.value - value = super().convert(value=value, param=param, ctx=ctx) - if value is None: - return None - return self.enum_type(value) +__all__ = ["ColoredCommand", "ColoredGroup"] class _OptionHighlighter(RegexHighlighter): diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index a106bc82..59228657 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -55,6 +55,7 @@ if TYPE_CHECKING: from _pytask.models import NodeInfo from _pytask.session import Session + from _pytask.settings import Settings @hookimpl @@ -104,7 +105,7 @@ def _collect_from_paths(session: Session) -> None: def _collect_from_tasks(session: Session) -> None: """Collect tasks from user provided tasks via the functional interface.""" - for raw_task in session.attrs["tasks"]: + for raw_task in session.attrs.get("tasks", []): if is_task_function(raw_task): if not hasattr(raw_task, "pytask_meta"): raw_task = task_decorator()(raw_task) # noqa: PLW2901 @@ -173,9 +174,9 @@ def _collect_not_collected_tasks(session: Session) -> None: @hookimpl -def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: +def pytask_ignore_collect(path: Path, config: Settings) -> bool: """Ignore a path during the collection.""" - return any(path.match(pattern) for pattern in config["ignore"]) + return any(path.match(pattern) for pattern in config.common.ignore) @hookimpl diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 4d659178..5f8a5fab 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -31,41 +31,42 @@ from _pytask.path import find_common_ancestor from _pytask.path import relative_to from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings_utils import SettingsBuilder +from _pytask.settings_utils import update_settings from _pytask.tree_util import tree_leaves if TYPE_CHECKING: from pathlib import Path from typing import NoReturn + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder + @ts.settings class Collect: nodes: bool = ts.option( default=False, help="Show a task's dependencies and products.", - click={"is_flag": True}, + click={"is_flag": True, "param_decls": ["--nodes"]}, ) @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - settings_builders["collect"] = SettingsBuilder(name="collect", function=collect) + settings_builder.commands["collect"] = collect + settings_builder.option_groups["collect"] = Collect() -def collect(**raw_config: Any | None) -> NoReturn: +def collect(settings: Settings, **arguments: Any) -> NoReturn: """Collect tasks and report information about them.""" - pm = storage.get() - raw_config["command"] = "collect" + settings = update_settings(settings, arguments) + pm = settings.common.pm try: - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + config = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session(config=config, hook=config.common.pm.hook) except (ConfigurationError, Exception): # pragma: no cover session = Session(config=config, exit_code=ExitCode.CONFIGURATION_FAILED) diff --git a/src/_pytask/config_utils.py b/src/_pytask/config_utils.py index b90cb9e4..0c92498e 100644 --- a/src/_pytask/config_utils.py +++ b/src/_pytask/config_utils.py @@ -19,22 +19,6 @@ __all__ = ["find_project_root_and_config", "read_config"] -def consolidate_settings_and_arguments(settings: Any, arguments: dict[str, Any]) -> Any: - """Consolidate the settings and the values from click arguments. - - Values from the command line have precedence over the settings from the - configuration file or from environment variables. Thus, we just plug in the values - from the command line into the settings. - - """ - for key, value in arguments.items(): - # We do not want to overwrite the settings with None or empty tuples that come - # from ``multiple=True`` The default is handled by the settings class. - if value is not None and value != (): - setattr(settings, key, value) - return settings - - def find_project_root_and_config( paths: Sequence[Path] | None, ) -> tuple[Path, Path | None]: diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index 6b0676e9..2d336c93 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -15,7 +15,6 @@ import typed_settings as ts from rich.text import Text -from _pytask.click import EnumChoice from _pytask.compat import check_for_optional_program from _pytask.compat import import_optional_dependency from _pytask.console import console @@ -66,19 +65,14 @@ class Dag: default=_RankDirection.TB, help="The direction of the directed graph. It can be ordered from top to " "bottom, TB, left to right, LR, bottom to top, BT, or right to left, RL.", - click={ - "type": EnumChoice(_RankDirection), - "param_decls": ["-r", "--rank-direction"], - }, + click={"param_decls": ["-r", "--rank-direction"]}, ) @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - settings_builders["dag"] = SettingsBuilder(name="dag", function=dag) + settings_builder.commands["dag"] = dag def dag(**raw_config: Any) -> int: @@ -86,7 +80,7 @@ def dag(**raw_config: Any) -> int: try: pm = storage.get() config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + session = Session(config=config, hook=config.common.pm.hook) except (ConfigurationError, Exception): # pragma: no cover console.print_exception() diff --git a/src/_pytask/data_catalog.py b/src/_pytask/data_catalog.py index 615fc021..a3a40711 100644 --- a/src/_pytask/data_catalog.py +++ b/src/_pytask/data_catalog.py @@ -10,7 +10,7 @@ import inspect import pickle from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING from attrs import define from attrs import field @@ -25,6 +25,9 @@ from _pytask.pluginmanager import storage from _pytask.session import Session +if TYPE_CHECKING: + from _pytask.settings import Settings + __all__ = ["DataCatalog"] @@ -63,7 +66,7 @@ class DataCatalog: entries: dict[str, PNode | PProvisionalNode] = field(factory=dict) name: str = "default" path: Path | None = None - _session_config: dict[str, Any] = field( + _session_config: Settings = field( factory=lambda *x: {"check_casing_of_paths": True} # noqa: ARG005 ) _instance_path: Path = field(factory=_get_parent_path_of_data_catalog_module) diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index a7d0515a..1e9df8d7 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -36,17 +36,16 @@ def _pdbcls_callback( ctx: click.Context, # noqa: ARG001 name: str, # noqa: ARG001 value: str | None, -) -> tuple[str, ...]: +) -> tuple[str, str] | None: """Validate the debugger class string passed to pdbcls.""" message = "'pdbcls' must be like IPython.terminal.debugger:TerminalPdb" - if value is None: - return () + return None if isinstance(value, str): split = value.split(":") if len(split) != 2: # noqa: PLR2004 raise click.BadParameter(message) - return tuple(split) + return (split[0], split[1]) raise click.BadParameter(message) @@ -57,8 +56,8 @@ class Debugging: click={"param_decls": ("--pdb",)}, help="Start the interactive debugger on errors.", ) - pdbcls: tuple[str, ...] = ts.option( - default=(), + pdbcls: tuple[str, str] | None = ts.option( + default=None, click={ "param_decls": ("--pdb-cls",), "metavar": "module_name:class_name", @@ -77,11 +76,9 @@ class Debugging: @hookimpl -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend command line interface.""" - settings_builders["build"].option_groups["debugging"] = Debugging() + settings_builder.option_groups["debugging"] = Debugging() @hookimpl(trylast=True) @@ -124,7 +121,7 @@ class PytaskPDB: _config: Settings | None = None _saved: ClassVar[list[tuple[Any, ...]]] = [] _recursive_debug: int = 0 - _wrapped_pdb_cls: tuple[tuple[str, ...], type[pdb.Pdb]] | None = None + _wrapped_pdb_cls: tuple[tuple[str, str] | None, type[pdb.Pdb]] | None = None @classmethod def _is_capturing(cls, capman: CaptureManager) -> bool: diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 2a616eca..90ef7c6b 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -50,9 +50,7 @@ def pytask_add_hooks(pm: PluginManager) -> None: @hookspec(historic=True) -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface. The hook can be used to extend the command line interface either by providing new @@ -120,7 +118,7 @@ def pytask_collect(session: Session) -> Any: @hookspec(firstresult=True) -def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: +def pytask_ignore_collect(path: Path, config: Settings) -> bool: """Ignore collected path. This hook is indicates for each directory and file whether it should be ignored. diff --git a/src/_pytask/live.py b/src/_pytask/live.py index 87ac0c48..d64c5c3e 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -50,11 +50,9 @@ class Live: @hookimpl -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend command line interface.""" - settings_builders["build"].option_groups["live"] = Live() + settings_builder.option_groups["live"] = Live() @hookimpl diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index 53d66f4b..5e9cf87f 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -55,10 +55,8 @@ class Logging: @hookimpl -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: - settings_builders["build"].option_groups["logging"] = Logging() +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: + settings_builder.option_groups["logging"] = Logging() @hookimpl diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index eca3f859..ff682cbb 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -7,6 +7,7 @@ from typing import AbstractSet from typing import Any +import typed_settings as ts from attrs import define from rich.table import Table @@ -21,11 +22,8 @@ from _pytask.mark.structures import MarkGenerator from _pytask.outcomes import ExitCode from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings import Markers -from _pytask.settings import Settings -from _pytask.settings_utils import SettingsBuilder +from _pytask.settings_utils import update_settings from _pytask.shared import parse_markers if TYPE_CHECKING: @@ -34,6 +32,8 @@ import networkx as nx from _pytask.node_protocols import PTask + from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder __all__ = [ @@ -50,14 +50,39 @@ ] -def markers(**raw_config: Any) -> NoReturn: +@ts.settings +class Markers: + """Settings for markers.""" + + strict_markers: bool = ts.option( + default=False, + click={"param_decls": ["--strict-markers"], "is_flag": True}, + help="Raise errors for unknown markers.", + ) + markers: dict[str, str] = ts.option(factory=dict, click={"hidden": True}) + marker_expression: str = ts.option( + default="", + click={ + "param_decls": ["-m", "marker_expression"], + "metavar": "MARKER_EXPRESSION", + }, + help="Select tasks via marker expressions.", + ) + expression: str = ts.option( + default="", + click={"param_decls": ["-k", "expression"], "metavar": "EXPRESSION"}, + help="Select tasks via expressions on task ids.", + ) + + +def markers_command(settings: Settings, **arguments: Any) -> NoReturn: """Show all registered markers.""" - raw_config["command"] = "markers" - pm = storage.get() + settings = update_settings(settings, arguments) + pm = settings.common.pm try: - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + config = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session(config=config, hook=config.common.pm.hook) except (ConfigurationError, Exception): # pragma: no cover console.print_exception() @@ -76,13 +101,10 @@ def markers(**raw_config: Any) -> NoReturn: @hookimpl -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Add marker related options.""" - settings_builders["markers"] = SettingsBuilder(name="markers", function=markers) - for settings_builder in settings_builders.values(): - settings_builder.option_groups["markers"] = Markers() + settings_builder.commands["markers"] = markers_command + settings_builder.option_groups["markers"] = Markers() @hookimpl diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index bf505ddb..03901ddf 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -153,11 +153,18 @@ class Common: ) def __attrs_post_init__(self) -> None: - self.root = ( - self.config_file.parent - if self.config_file - else Path(os.path.commonpath(self.paths)) - ) + # Set self.root. + if self.config_file: + self.root = self.config_file.parent + elif self.paths: + candidate = Path(os.path.commonpath(self.paths)) + if candidate.is_dir(): + self.root = candidate + else: + self.root = candidate.parent + else: + self.root = Path.cwd() + self.cache = self.root / ".pytask" @@ -170,10 +177,7 @@ def __attrs_post_init__(self) -> None: @hookimpl(trylast=True) -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Register general markers.""" - for settings_builder in settings_builders.values(): - settings_builder.arguments.append(_PATH_ARGUMENT) - settings_builder.option_groups["common"] = Common() + settings_builder.option_groups["common"] = Common() + settings_builder.arguments.append(_PATH_ARGUMENT) diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index f988a7ec..3159915c 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -38,12 +38,12 @@ def pytask_add_hooks(pm: PluginManager) -> None: builtin_hook_impl_modules = ( "_pytask.build", "_pytask.capture", - # "_pytask.clean", + "_pytask.clean", "_pytask.collect", - # "_pytask.collect_command", + "_pytask.collect_command", "_pytask.config", "_pytask.dag", - # "_pytask.dag_command", + "_pytask.dag_command", "_pytask.database", "_pytask.debugging", "_pytask.execute", @@ -53,7 +53,7 @@ def pytask_add_hooks(pm: PluginManager) -> None: "_pytask.nodes", "_pytask.parameters", "_pytask.persist", - # "_pytask.profile", + "_pytask.profile", "_pytask.skipping", "_pytask.task", "_pytask.warnings", diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 12f8f829..893a769c 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -7,17 +7,16 @@ import sys import time from contextlib import suppress +from enum import Enum from typing import TYPE_CHECKING from typing import Any from typing import Generator -import click +import typed_settings as ts from rich.table import Table from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column -from _pytask.click import ColoredCommand -from _pytask.click import EnumChoice from _pytask.console import console from _pytask.console import format_task_name from _pytask.dag import create_dag @@ -30,10 +29,8 @@ from _pytask.outcomes import ExitCode from _pytask.outcomes import TaskOutcome from _pytask.pluginmanager import hookimpl -from _pytask.pluginmanager import storage from _pytask.session import Session -from _pytask.settings import _ExportFormats -from _pytask.settings_utils import SettingsBuilder +from _pytask.settings_utils import update_settings from _pytask.traceback import Traceback if TYPE_CHECKING: @@ -42,6 +39,21 @@ from _pytask.reports import ExecutionReport from _pytask.settings import Settings + from _pytask.settings_utils import SettingsBuilder + + +class _ExportFormats(Enum): + NO = "no" + JSON = "json" + CSV = "csv" + + +@ts.settings +class Profile: + export: _ExportFormats = ts.option( + default=_ExportFormats.NO, + help="Export the profile in the specified format.", + ) class Runtime(BaseTable): @@ -55,11 +67,10 @@ class Runtime(BaseTable): @hookimpl(tryfirst=True) -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the command line interface.""" - settings_builders["profile"] = SettingsBuilder(name="profile", function=profile) + settings_builder.commands["profile"] = profile_command + settings_builder.option_groups["profile"] = Profile() @hookimpl @@ -103,21 +114,14 @@ def _create_or_update_runtime(task_signature: str, start: float, end: float) -> session.commit() -@click.command(cls=ColoredCommand) -@click.option( - "--export", - type=EnumChoice(_ExportFormats), - default=_ExportFormats.NO, - help="Export the profile in the specified format.", -) -def profile(**raw_config: Any) -> NoReturn: +def profile_command(settings: Settings, **arguments: Any) -> NoReturn: """Show information about tasks like runtime and memory consumption of products.""" - pm = storage.get() - raw_config["command"] = "profile" + settings = update_settings(settings, arguments) + pm = settings.common.pm try: - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + config = pm.hook.pytask_configure(pm=pm, config=settings) + session = Session(config=config, hook=config.common.pm.hook) except (ConfigurationError, Exception): # pragma: no cover session = Session(exit_code=ExitCode.CONFIGURATION_FAILED) diff --git a/src/_pytask/session.py b/src/_pytask/session.py index 022003c8..e0c32ad2 100644 --- a/src/_pytask/session.py +++ b/src/_pytask/session.py @@ -73,6 +73,9 @@ class Session: warnings: list[WarningReport] = field(factory=list) @classmethod - def from_config(cls, config: Settings) -> Session: + def from_config(cls, config: dict[str, Any]) -> Session: """Construct the class from a config.""" - return cls(config=config, hook=config.common.pm.hook) + from _pytask.cli import settings_builder + + settings = settings_builder.load_settings(kwargs=config) + return cls(config=settings, hook=settings.common.pm.hook) diff --git a/src/_pytask/settings.py b/src/_pytask/settings.py index 9d926ae1..425d8653 100644 --- a/src/_pytask/settings.py +++ b/src/_pytask/settings.py @@ -1,150 +1,8 @@ from __future__ import annotations -from enum import Enum - import typed_settings as ts -from _pytask.capture_utils import CaptureMethod -from _pytask.capture_utils import ShowCapture -from _pytask.click import EnumChoice - -__all__ = ["Build", "Capture", "Clean", "Markers", "Profile", "Settings"] - - -@ts.settings -class Build: - stop_after_first_failure: bool = ts.option( - default=False, - click={"param_decls": ("-x", "--stop-after-first-failure"), "is_flag": True}, - help="Stop after the first failure.", - ) - max_failures: float = ts.option( - default=float("inf"), - click={"param_decls": ("--max-failures",)}, - help="Stop after some failures.", - ) - show_errors_immediately: bool = ts.option( - default=False, - click={"param_decls": ("--show-errors-immediately",), "is_flag": True}, - help="Show errors with tracebacks as soon as the task fails.", - ) - show_traceback: bool = ts.option( - default=True, - click={"param_decls": ("--show-traceback", "--show-no-traceback")}, - help="Choose whether tracebacks should be displayed or not.", - ) - dry_run: bool = ts.option( - default=False, - click={"param_decls": ("--dry-run",), "is_flag": True}, - help="Perform a dry-run.", - ) - force: bool = ts.option( - default=False, - click={"param_decls": ("-f", "--force"), "is_flag": True}, - help="Execute a task even if it succeeded successfully before.", - ) - check_casing_of_paths: bool = ts.option( - default=True, - click={"param_decls": ("--check-casing-of-paths",), "hidden": True}, - ) - - -@ts.settings -class Capture: - """Settings for capturing.""" - - capture: CaptureMethod = ts.option( - default=CaptureMethod.FD, - click={"param_decls": ["--capture"]}, - help="Per task capturing method.", - ) - s: bool = ts.option( - default=False, - click={"param_decls": ["-s"], "is_flag": True}, - help="Shortcut for --capture=no.", - ) - show_capture: ShowCapture = ts.option( - default=ShowCapture.ALL, - click={"param_decls": ["--show-capture"]}, - help="Choose which captured output should be shown for failed tasks.", - ) - - -class _CleanMode(Enum): - DRY_RUN = "dry-run" - FORCE = "force" - INTERACTIVE = "interactive" - - -@ts.settings -class Clean: - directories: bool = ts.option( - default=False, - help="Remove whole directories.", - click={"is_flag": True, "param_decls": ["-d", "--directories"]}, - ) - exclude: tuple[str, ...] = ts.option( - factory=tuple, - help="A filename pattern to exclude files from the cleaning process.", - click={ - "multiple": True, - "metavar": "PATTERN", - "param_decls": ["-e", "--exclude"], - }, - ) - mode: _CleanMode = ts.option( - default=_CleanMode.DRY_RUN, - help=( - "Choose 'dry-run' to print the paths of files/directories which would be " - "removed, 'interactive' for a confirmation prompt for every path, and " - "'force' to remove all unknown paths at once." - ), - click={"type": EnumChoice(_CleanMode), "param_decls": ["-m", "--mode"]}, - ) - quiet: bool = ts.option( - default=False, - help="Do not print the names of the removed paths.", - click={"is_flag": True, "param_decls": ["-q", "--quiet"]}, - ) - - -@ts.settings -class Markers: - """Settings for markers.""" - - strict_markers: bool = ts.option( - default=False, - click={"param_decls": ["--strict-markers"], "is_flag": True}, - help="Raise errors for unknown markers.", - ) - markers: dict[str, str] = ts.option(factory=dict, click={"hidden": True}) - marker_expression: str = ts.option( - default="", - click={ - "param_decls": ["-m", "marker_expression"], - "metavar": "MARKER_EXPRESSION", - }, - help="Select tasks via marker expressions.", - ) - expression: str = ts.option( - default="", - click={"param_decls": ["-k", "expression"], "metavar": "EXPRESSION"}, - help="Select tasks via expressions on task ids.", - ) - - -class _ExportFormats(Enum): - NO = "no" - JSON = "json" - CSV = "csv" - - -@ts.settings -class Profile: - export: _ExportFormats = ts.option( - default=_ExportFormats.NO, - help="Export the profile in the specified format.", - ) +__all__ = ["Settings"] @ts.settings diff --git a/src/_pytask/settings.pyi b/src/_pytask/settings.pyi index 60167d08..14c76b97 100644 --- a/src/_pytask/settings.pyi +++ b/src/_pytask/settings.pyi @@ -1,55 +1,17 @@ -from enum import Enum - -from _pytask.capture_utils import CaptureMethod -from _pytask.capture_utils import ShowCapture +from _pytask.build import Build +from _pytask.capture import Capture +from _pytask.clean import Clean from _pytask.collect_command import Collect from _pytask.dag_command import Dag from _pytask.debugging import Debugging from _pytask.live import Live from _pytask.logging import Logging +from _pytask.mark import Markers from _pytask.parameters import Common +from _pytask.profile import Profile from _pytask.warnings import Warnings -__all__ = ["Build", "Capture", "Clean", "Markers", "Profile", "Settings"] - -class Build: - stop_after_first_failure: bool - max_failures: float - show_errors_immediately: bool - show_traceback: bool - dry_run: bool - force: bool - check_casing_of_paths: bool - -class _ExportFormats(Enum): - NO: str - JSON: str - CSV: str - -class Profile: - export: _ExportFormats - -class _CleanMode(Enum): - DRY_RUN: str - FORCE: str - INTERACTIVE: str - -class Clean: - directories: bool - exclude: tuple[str, ...] - mode: _CleanMode - quiet: bool - -class Markers: - strict_markers: bool - markers: dict[str, str] - marker_expression: str - expression: str - -class Capture: - capture: CaptureMethod - s: bool - show_capture: ShowCapture +__all__ = ["Settings"] class Settings: build: Build diff --git a/src/_pytask/settings_utils.py b/src/_pytask/settings_utils.py index e9f809f5..308f3fe0 100644 --- a/src/_pytask/settings_utils.py +++ b/src/_pytask/settings_utils.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +from enum import Enum from typing import TYPE_CHECKING from typing import Any from typing import Callable @@ -11,14 +12,16 @@ import typed_settings as ts from attrs import define from attrs import field +from pluggy import PluginManager from typed_settings.cli_click import OptionGroupFactory from typed_settings.exceptions import ConfigFileLoadError from typed_settings.exceptions import ConfigFileNotFoundError +from typed_settings.types import LoadedSettings +from typed_settings.types import LoaderMeta from typed_settings.types import OptionList from typed_settings.types import SettingsClass from typed_settings.types import SettingsDict -from _pytask.click import ColoredCommand from _pytask.console import console from _pytask.settings import Settings @@ -36,24 +39,46 @@ __all__ = ["SettingsBuilder", "TomlFormat"] +def _handle_enum(type: type[Enum], default: Any, is_optional: bool) -> dict[str, Any]: # noqa: A002 + """Use enum values as choices for click options.""" + kwargs = {"type": click.Choice([i.value for i in type])} + if isinstance(default, type): + kwargs["default"] = default.value + elif is_optional: + kwargs["default"] = None + return kwargs + + @define class SettingsBuilder: - name: str - function: Callable[..., Any] + commands: dict[str, Callable[..., Any]] = field(factory=dict) option_groups: dict[str, Any] = field(factory=dict) arguments: list[Any] = field(factory=list) def build_settings(self) -> Any: return ts.combine("Settings", Settings, self.option_groups) # type: ignore[arg-type] - def build_command(self, loaders: list[Loader]) -> Any: + def load_settings(self, kwargs: dict[str, Any]) -> Any: + settings = self.build_settings() + return ts.load_settings( + settings, + create_settings_loaders(kwargs=kwargs), + converter=create_converter(), + ) + + def build_decorator(self) -> Any: settings = self.build_settings() - command = ts.click_options( - settings, loaders, decorator_factory=OptionGroupFactory() - )(self.function) - command = click.command(name=self.name, cls=ColoredCommand)(command) - command.params.extend(self.arguments) - return command + + type_dict = {**ts.cli_click.DEFAULT_TYPES, Enum: _handle_enum} + type_handler = ts.cli_click.ClickHandler(type_dict) + + return ts.click_options( + settings, + create_settings_loaders(), + converter=create_converter(), + decorator_factory=OptionGroupFactory(), + type_args_maker=ts.cli_utils.TypeArgsMaker(type_handler), + ) _ALREADY_PRINTED_DEPRECATION_MSG: bool = False @@ -122,34 +147,51 @@ def __call__( console.print(self.deprecated) settings["common.config_file"] = path settings["common.root"] = path.parent - settings = self._rewrite_paths_of_options(settings, options) + settings = _rewrite_paths_of_options(settings, options) return cast(SettingsDict, settings) - def _rewrite_paths_of_options( - self, settings: SettingsDict, options: OptionList - ) -> SettingsDict: - """Rewrite paths of options in the settings.""" - option_paths = {option.path for option in options} - for name in list(settings): - if name in option_paths: - continue - for option_path in option_paths: - if name in option_path: - settings[option_path] = settings.pop(name) - break +class DictLoader: + """Load settings from a dict of values.""" + + def __init__(self, settings: dict) -> None: + self.settings = settings - return settings + def __call__( + self, + settings_cls: SettingsClass, # noqa: ARG002 + options: OptionList, + ) -> LoadedSettings: + settings = _rewrite_paths_of_options(self.settings, options) + nested_settings = {name.split(".")[0]: {} for name in settings} + for long_name, value in settings.items(): + group, name = long_name.split(".") + nested_settings[group][name] = value + return LoadedSettings(nested_settings, LoaderMeta(self)) -def load_settings(settings_cls: Any) -> Any: +def load_settings(settings_cls: Any, kwargs: dict[str, Any] | None = None) -> Any: """Load the settings.""" - loaders = create_settings_loaders() - return ts.load_settings(settings_cls, loaders) + loaders = create_settings_loaders(kwargs=kwargs) + converter = create_converter() + return ts.load_settings(settings_cls, loaders, converter=converter) -def create_settings_loaders() -> list[Loader]: +def create_converter() -> ts.Converter: + """Create the converter.""" + converter = ts.converters.get_default_ts_converter() + converter.scalar_converters[Enum] = ( + lambda val, cls: val if isinstance(val, cls) else cls(val) + ) + converter.scalar_converters[PluginManager] = ( + lambda val, cls: val if isinstance(val, cls) else cls(**val) + ) + return converter + + +def create_settings_loaders(kwargs: dict[str, Any] | None = None) -> list[Loader]: """Create the loaders for the settings.""" + kwargs_ = kwargs or {} return [ ts.FileLoader( files=[ts.find("pyproject.toml")], @@ -173,6 +215,7 @@ def create_settings_loaders() -> list[Loader]: }, ), ts.EnvLoader(prefix="PYTASK_", nested_delimiter="_"), + DictLoader(kwargs_), ] @@ -180,14 +223,34 @@ def update_settings(settings: Any, updates: dict[str, Any]) -> Any: """Update the settings recursively with some updates.""" names = [i for i in dir(settings) if not i.startswith("_")] for name in names: + if attrs.has(getattr(settings, name)): + update_settings(getattr(settings, name), updates) + continue + if name in updates: value = updates[name] if value in ((), []): continue - setattr(settings, name, updates[name]) + return settings - if attrs.has(getattr(settings, name)): - update_settings(getattr(settings, name), updates) - return settings +def convert_settings_to_kwargs(settings: Settings) -> dict[str, Any]: + """Convert the settings to kwargs.""" + kwargs = {} + names = [i for i in dir(settings) if not i.startswith("_")] + for name in names: + kwargs = kwargs | attrs.asdict(getattr(settings, name)) + return kwargs + + +def _rewrite_paths_of_options( + settings: SettingsDict, options: OptionList +) -> SettingsDict: + """Rewrite paths of options in the settings.""" + option_name_to_path = {option.path.split(".")[1]: option.path for option in options} + return { + option_name_to_path[name]: value + for name, value in settings.items() + if name in option_name_to_path + } diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py index 41394020..809988b9 100644 --- a/src/_pytask/traceback.py +++ b/src/_pytask/traceback.py @@ -51,7 +51,7 @@ class Traceback: _show_locals: ClassVar[bool] = False suppress: ClassVar[tuple[Path, ...]] = ( _PLUGGY_DIRECTORY, - # _PYTASK_DIRECTORY, + _PYTASK_DIRECTORY, TREE_UTIL_LIB_DIRECTORY, ) @@ -134,8 +134,8 @@ def _is_internal_or_hidden_traceback_frame( exc_info: ExceptionInfo, suppress: tuple[Path, ...] = ( _PLUGGY_DIRECTORY, + # _PYTASK_DIRECTORY, TREE_UTIL_LIB_DIRECTORY, - _PYTASK_DIRECTORY, ), ) -> bool: """Return ``True`` if traceback frame belongs to internal packages or is hidden. diff --git a/src/_pytask/warnings.py b/src/_pytask/warnings.py index 79aa18dd..eb2040cc 100644 --- a/src/_pytask/warnings.py +++ b/src/_pytask/warnings.py @@ -44,11 +44,9 @@ class Warnings: @hookimpl -def pytask_extend_command_line_interface( - settings_builders: dict[str, SettingsBuilder], -) -> None: +def pytask_extend_command_line_interface(settings_builder: SettingsBuilder) -> None: """Extend the cli.""" - settings_builders["build"].option_groups["warnings"] = Warnings() + settings_builder.option_groups["warnings"] = Warnings() @hookimpl diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index c032aa59..d010a914 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -11,7 +11,6 @@ from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup -from _pytask.click import EnumChoice from _pytask.collect_utils import parse_dependencies_from_task_function from _pytask.collect_utils import parse_products_from_task_function from _pytask.compat import check_for_optional_program @@ -97,7 +96,6 @@ "DataCatalog", "DatabaseSession", "DirectoryNode", - "EnumChoice", "ExecutionError", "ExecutionReport", "Exit", diff --git a/tests/test_click.py b/tests/test_click.py index ec01e758..f6ee1a79 100644 --- a/tests/test_click.py +++ b/tests/test_click.py @@ -1,10 +1,6 @@ from __future__ import annotations -import enum - -import click import pytest -from pytask import EnumChoice from pytask import cli @@ -19,38 +15,3 @@ def test_choices_are_displayed_in_help_page(runner): def test_defaults_are_displayed(runner): result = runner.invoke(cli, ["build", "--help"]) assert "[default: all]" in result.output - - -@pytest.mark.unit() -@pytest.mark.parametrize("method", ["first", "second"]) -def test_enum_choice(runner, method): - class Method(enum.Enum): - FIRST = "first" - SECOND = "second" - - @click.command() - @click.option("--method", type=EnumChoice(Method)) - def test(method): - print(f"method={method}") # noqa: T201 - - result = runner.invoke(test, ["--method", method]) - - assert result.exit_code == 0 - assert f"method=Method.{method.upper()}" in result.output - - -@pytest.mark.unit() -def test_enum_choice_error(runner): - class Method(enum.Enum): - FIRST = "first" - SECOND = "second" - - @click.command() - @click.option("--method", type=EnumChoice(Method)) - def test(): ... - - result = runner.invoke(test, ["--method", "third"]) - - assert result.exit_code == 2 - assert "Invalid value for '--method': " in result.output - assert "'third' is not one of 'first', 'second'." in result.output From f7ba983cb5f607fe4328d5dc2226d7874f537b1d Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 21 Apr 2024 12:06:11 +0200 Subject: [PATCH 09/15] Preliminary solution for build function. --- src/_pytask/build.py | 70 +++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 781affc6..28e437ae 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -13,8 +13,6 @@ import typed_settings as ts -from _pytask.capture_utils import CaptureMethod -from _pytask.capture_utils import ShowCapture from _pytask.console import console from _pytask.dag import create_dag from _pytask.exceptions import CollectionError @@ -28,11 +26,15 @@ from _pytask.settings_utils import SettingsBuilder from _pytask.settings_utils import update_settings from _pytask.traceback import Traceback +from _pytask.typing import NoDefault +from _pytask.typing import no_default if TYPE_CHECKING: from pathlib import Path from typing import NoReturn + from _pytask.capture_utils import CaptureMethod + from _pytask.capture_utils import ShowCapture from _pytask.node_protocols import PTask from _pytask.settings import Settings @@ -102,35 +104,42 @@ def pytask_unconfigure(session: Session) -> None: def build( # noqa: PLR0913 *, - capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.FD, - check_casing_of_paths: bool = True, - debug_pytask: bool = False, - disable_warnings: bool = False, - dry_run: bool = False, + capture: Literal["fd", "no", "sys", "tee-sys"] + | CaptureMethod + | NoDefault = no_default, + check_casing_of_paths: bool | NoDefault = no_default, + debug_pytask: bool | NoDefault = no_default, + disable_warnings: bool | NoDefault = no_default, + dry_run: bool | NoDefault = no_default, editor_url_scheme: Literal["no_link", "file", "vscode", "pycharm"] # noqa: PYI051 - | str = "file", - expression: str = "", - force: bool = False, - ignore: Iterable[str] = (), - marker_expression: str = "", - max_failures: float = float("inf"), - n_entries_in_table: int = 15, - paths: Path | Iterable[Path] = (), - pdb: bool = False, - pdb_cls: str = "", - s: bool = False, + | str + | NoDefault = no_default, + expression: str | NoDefault = no_default, + force: bool | NoDefault = no_default, + ignore: Iterable[str] | NoDefault = no_default, + marker_expression: str | NoDefault = no_default, + max_failures: float | NoDefault = no_default, + n_entries_in_table: int | NoDefault = no_default, + paths: Path | Iterable[Path] | NoDefault = no_default, + pdb: bool | NoDefault = no_default, + pdb_cls: str | NoDefault = no_default, + s: bool | NoDefault = no_default, show_capture: Literal["no", "stdout", "stderr", "all"] - | ShowCapture = ShowCapture.ALL, - show_errors_immediately: bool = False, - show_locals: bool = False, - show_traceback: bool = True, - sort_table: bool = True, - stop_after_first_failure: bool = False, - strict_markers: bool = False, - tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (), - task_files: Iterable[str] = ("task_*.py",), - trace: bool = False, - verbose: int = 1, + | ShowCapture + | NoDefault = no_default, + show_errors_immediately: bool | NoDefault = no_default, + show_locals: bool | NoDefault = no_default, + show_traceback: bool | NoDefault = no_default, + sort_table: bool | NoDefault = no_default, + stop_after_first_failure: bool | NoDefault = no_default, + strict_markers: bool | NoDefault = no_default, + tasks: Callable[..., Any] + | PTask + | Iterable[Callable[..., Any] | PTask] + | NoDefault = no_default, + task_files: Iterable[str] | NoDefault = no_default, + trace: bool | NoDefault = no_default, + verbose: int | NoDefault = no_default, **kwargs: Any, ) -> Session: """Run pytask. @@ -239,10 +248,11 @@ def build( # noqa: PLR0913 "verbose": verbose, **kwargs, } + filtered_updates = {k: v for k, v in updates.items() if v is not no_default} from _pytask.cli import settings_builder - settings = settings_builder.load_settings(kwargs=updates) + settings = settings_builder.load_settings(kwargs=filtered_updates) except (ConfigurationError, Exception): console.print(Traceback(sys.exc_info())) session = Session(exit_code=ExitCode.CONFIGURATION_FAILED) From 6070085fda9c4dc0f864bb48052f8e02c5911773 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 21 Apr 2024 12:06:45 +0200 Subject: [PATCH 10/15] Rename config_file to config. --- src/_pytask/clean.py | 4 ++-- src/_pytask/logging.py | 4 ++-- src/_pytask/parameters.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index f3a6bbfe..7e2f5ff4 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -182,8 +182,8 @@ def _collect_all_paths_known_to_pytask(session: Session) -> set[Path]: known_paths = known_files | known_directories - if session.config.common.config_file: - known_paths.add(session.config.common.config_file) + if session.config.common.config: + known_paths.add(session.config.common.config) known_paths.add(session.config.common.root) known_paths.add(session.config.common.cache / "pytask.sqlite3") diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index 5e9cf87f..ce1458fc 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -94,8 +94,8 @@ def pytask_log_session_header(session: Session) -> None: f"pytask {_pytask.__version__}, pluggy {pluggy.__version__}" ) console.print(f"Root: {session.config.common.root}") - if session.config.common.config_file is not None: - console.print(f"Configuration: {session.config.common.config_file}") + if session.config.common.config is not None: + console.print(f"Configuration: {session.config.common.config}") plugin_info = session.config.common.pm.list_plugin_distinfo() if plugin_info: diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index 03901ddf..c65205f3 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -91,7 +91,7 @@ class Common: """Common settings for the command line interface.""" cache: Path = ts.option(init=False, click={"hidden": True}) - config_file: Path | None = ts.option( + config: Path | None = ts.option( default=None, click={"param_decls": ["--config-file"], "hidden": True} ) debug_pytask: bool = ts.option( @@ -154,8 +154,8 @@ class Common: def __attrs_post_init__(self) -> None: # Set self.root. - if self.config_file: - self.root = self.config_file.parent + if self.config: + self.root = self.config.parent elif self.paths: candidate = Path(os.path.commonpath(self.paths)) if candidate.is_dir(): From f5238da3e5698df9a80380f219694b9c2acc1536 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 21 Apr 2024 14:37:57 +0200 Subject: [PATCH 11/15] fix. --- src/_pytask/build.py | 8 +++----- src/_pytask/clean.py | 4 ++-- src/_pytask/logging.py | 4 ++-- src/_pytask/parameters.py | 8 ++++---- src/_pytask/traceback.py | 2 +- tests/conftest.py | 11 +++++++++++ tests/test_collect.py | 12 +++++++----- tests/test_database.py | 25 ------------------------- 8 files changed, 30 insertions(+), 44 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 28e437ae..f939e7d6 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -25,6 +25,7 @@ from _pytask.session import Session from _pytask.settings_utils import SettingsBuilder from _pytask.settings_utils import update_settings +from _pytask.shared import to_list from _pytask.traceback import Traceback from _pytask.typing import NoDefault from _pytask.typing import no_default @@ -133,10 +134,7 @@ def build( # noqa: PLR0913 sort_table: bool | NoDefault = no_default, stop_after_first_failure: bool | NoDefault = no_default, strict_markers: bool | NoDefault = no_default, - tasks: Callable[..., Any] - | PTask - | Iterable[Callable[..., Any] | PTask] - | NoDefault = no_default, + tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (), task_files: Iterable[str] | NoDefault = no_default, trace: bool | NoDefault = no_default, verbose: int | NoDefault = no_default, @@ -231,7 +229,7 @@ def build( # noqa: PLR0913 "marker_expression": marker_expression, "max_failures": max_failures, "n_entries_in_table": n_entries_in_table, - "paths": paths, + "paths": to_list(paths) if paths is not no_default else no_default, "pdb": pdb, "pdb_cls": pdb_cls, "s": s, diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index 7e2f5ff4..f3a6bbfe 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -182,8 +182,8 @@ def _collect_all_paths_known_to_pytask(session: Session) -> set[Path]: known_paths = known_files | known_directories - if session.config.common.config: - known_paths.add(session.config.common.config) + if session.config.common.config_file: + known_paths.add(session.config.common.config_file) known_paths.add(session.config.common.root) known_paths.add(session.config.common.cache / "pytask.sqlite3") diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index ce1458fc..5e9cf87f 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -94,8 +94,8 @@ def pytask_log_session_header(session: Session) -> None: f"pytask {_pytask.__version__}, pluggy {pluggy.__version__}" ) console.print(f"Root: {session.config.common.root}") - if session.config.common.config is not None: - console.print(f"Configuration: {session.config.common.config}") + if session.config.common.config_file is not None: + console.print(f"Configuration: {session.config.common.config_file}") plugin_info = session.config.common.pm.list_plugin_distinfo() if plugin_info: diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index c65205f3..b5d2848d 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -91,8 +91,8 @@ class Common: """Common settings for the command line interface.""" cache: Path = ts.option(init=False, click={"hidden": True}) - config: Path | None = ts.option( - default=None, click={"param_decls": ["--config-file"], "hidden": True} + config_file: Path | None = ts.option( + default=None, click={"param_decls": ["--config"], "hidden": True} ) debug_pytask: bool = ts.option( default=False, @@ -154,8 +154,8 @@ class Common: def __attrs_post_init__(self) -> None: # Set self.root. - if self.config: - self.root = self.config.parent + if self.config_file: + self.root = self.config_file.parent elif self.paths: candidate = Path(os.path.commonpath(self.paths)) if candidate.is_dir(): diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py index 809988b9..63ff3800 100644 --- a/src/_pytask/traceback.py +++ b/src/_pytask/traceback.py @@ -51,7 +51,7 @@ class Traceback: _show_locals: ClassVar[bool] = False suppress: ClassVar[tuple[Path, ...]] = ( _PLUGGY_DIRECTORY, - _PYTASK_DIRECTORY, + # _PYTASK_DIRECTORY, TREE_UTIL_LIB_DIRECTORY, ) diff --git a/tests/conftest.py b/tests/conftest.py index cf1f65d8..0c49a2c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re import sys from contextlib import contextmanager @@ -114,3 +115,13 @@ def pytest_collection_modifyitems(session, config, items) -> None: # noqa: ARG0 for item in items: if isinstance(item, NotebookItem): item.add_marker(pytest.mark.xfail(reason="The tests are flaky.")) + + +@contextmanager +def enter_directory(path: Path): + old_cwd = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(old_cwd) diff --git a/tests/test_collect.py b/tests/test_collect.py index 93591998..d10c4966 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -16,6 +16,8 @@ from pytask import build from pytask import cli +from tests.conftest import enter_directory + @pytest.mark.end_to_end() @pytest.mark.parametrize( @@ -111,7 +113,7 @@ def test_collect_same_task_different_ways(tmp_path, path_extension): ], ) def test_collect_files_w_custom_file_name_pattern( - tmp_path, task_files, pattern, expected_collected_tasks + runner, tmp_path, task_files, pattern, expected_collected_tasks ): tmp_path.joinpath("pyproject.toml").write_text( f"[tool.pytask.ini_options]\ntask_files = {pattern}" @@ -120,10 +122,10 @@ def test_collect_files_w_custom_file_name_pattern( for file_ in task_files: tmp_path.joinpath(file_).write_text("def task_example(): pass") - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.OK - assert len(session.tasks) == expected_collected_tasks + with enter_directory(tmp_path): + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert f"Collected {expected_collected_tasks} task" in result.output def test_error_with_invalid_file_name_pattern(runner, tmp_path): diff --git a/tests/test_database.py b/tests/test_database.py index 0745a999..cd1f7898 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -7,7 +7,6 @@ from pytask import ExitCode from pytask import State from pytask import build -from pytask import cli from pytask import create_database from pytask.path import hash_path from sqlalchemy.engine import make_url @@ -50,27 +49,3 @@ def task_write(path=Path("in.txt"), produces=Path("out.txt")): ): hash_ = db_session.get(State, (task_id, id_)).hash_ assert hash_ == hash_path(path, path.stat().st_mtime) - - -@pytest.mark.end_to_end() -def test_rename_database_w_config(tmp_path, runner): - """Modification dates of input and output files are stored in database.""" - path_to_db = tmp_path.joinpath(".db.sqlite") - tmp_path.joinpath("pyproject.toml").write_text( - "[tool.pytask.ini_options]\ndatabase_url='sqlite:///.db.sqlite'" - ) - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert path_to_db.exists() - - -@pytest.mark.end_to_end() -def test_rename_database_w_cli(tmp_path, runner): - """Modification dates of input and output files are stored in database.""" - path_to_db = tmp_path.joinpath(".db.sqlite") - result = runner.invoke( - cli, - ["--database-url", "sqlite:///.db.sqlite", tmp_path.as_posix()], - ) - assert result.exit_code == ExitCode.OK - assert path_to_db.exists() From 812779c44799c419a21f4f0db54cb144e402d495 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 23 Apr 2024 15:54:29 +0200 Subject: [PATCH 12/15] Temp --- pyproject.toml | 10 ++--- src/_pytask/cli.py | 15 +++++-- src/_pytask/settings_utils.py | 83 +++++++++++++++++++---------------- src/_pytask/traceback.py | 5 ++- tests/test_cli.py | 21 ++++++--- tests/test_config.py | 12 +++++ 6 files changed, 92 insertions(+), 54 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9c1848a7..7c02e1e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "sqlalchemy>=2", 'tomli>=1; python_version < "3.11"', 'typing-extensions; python_version < "3.9"', - "typed-settings[cattrs,option-groups]", + "typed-settings[option-groups]", "universal-pathlib>=0.2.2", ] @@ -222,8 +222,8 @@ wrap = 88 end_of_line = "keep" -# [tool.pytask] -# debug_pytask = false +[tool.pytask] +debug_pytask = 1 -# [tool.pytask.ini_options] -# debug_pytask = true +[tool.pytask.ini_options] +capture = "all" diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index 38ec2ee7..7561c6c2 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -2,8 +2,10 @@ from __future__ import annotations +import sys from typing import Any +from _pytask.traceback import Traceback import click from packaging.version import parse as parse_version @@ -11,6 +13,7 @@ from _pytask.click import ColoredGroup from _pytask.pluginmanager import storage from _pytask.settings_utils import SettingsBuilder +from _pytask.console import console _CONTEXT_SETTINGS: dict[str, Any] = { "help_option_names": ("-h", "--help"), @@ -49,7 +52,11 @@ def cli() -> None: """Manage your tasks with pytask.""" -for name, func in settings_builder.commands.items(): - command = click.command(name=name, cls=ColoredCommand)(decorator(func)) - command.params.extend(settings_builder.arguments) - cli.add_command(command) +try: + for name, func in settings_builder.commands.items(): + command = click.command(name=name, cls=ColoredCommand)(decorator(func)) + command.params.extend(settings_builder.arguments) + cli.add_command(command) +except Exception: # noqa: BLE001 + traceback = Traceback(sys.exc_info(), show_locals=False) + console.print(traceback) diff --git a/src/_pytask/settings_utils.py b/src/_pytask/settings_utils.py index 308f3fe0..18c6241d 100644 --- a/src/_pytask/settings_utils.py +++ b/src/_pytask/settings_utils.py @@ -85,12 +85,7 @@ def build_decorator(self) -> Any: class TomlFormat: - """ - Support for TOML files. Read settings from the given *section*. - - Args: - section: The config file section to load settings from. - """ + """Support for TOML files.""" def __init__( self, @@ -108,22 +103,7 @@ def __call__( settings_cls: SettingsClass, # noqa: ARG002 options: OptionList, ) -> SettingsDict: - """ - Load settings from a TOML file and return them as a dict. - - Args: - path: The path to the config file. - options: The list of available settings. - settings_cls: The base settings class for all options. If ``None``, load - top level settings. - - Return: - A dict with the loaded settings. - - Raise: - ConfigFileNotFoundError: If *path* does not exist. - ConfigFileLoadError: If *path* cannot be read/loaded/decoded. - """ + """Load settings from a TOML file and return them as a dict.""" try: with path.open("rb") as f: settings = tomllib.load(f) @@ -144,10 +124,10 @@ def __call__( global _ALREADY_PRINTED_DEPRECATION_MSG # noqa: PLW0603 if self.deprecated and not _ALREADY_PRINTED_DEPRECATION_MSG: _ALREADY_PRINTED_DEPRECATION_MSG = True - console.print(self.deprecated) + console.print(self.deprecated, style="skipped") settings["common.config_file"] = path settings["common.root"] = path.parent - settings = _rewrite_paths_of_options(settings, options) + settings = _rewrite_paths_of_options(settings, options, section=self.section) return cast(SettingsDict, settings) @@ -162,7 +142,7 @@ def __call__( settings_cls: SettingsClass, # noqa: ARG002 options: OptionList, ) -> LoadedSettings: - settings = _rewrite_paths_of_options(self.settings, options) + settings = _rewrite_paths_of_options(self.settings, options, section=None) nested_settings = {name.split(".")[0]: {} for name in settings} for long_name, value in settings.items(): group, name = long_name.split(".") @@ -177,12 +157,24 @@ def load_settings(settings_cls: Any, kwargs: dict[str, Any] | None = None) -> An return ts.load_settings(settings_cls, loaders, converter=converter) +def _convert_to_enum(val: Any, cls: type[Enum]) -> Enum: + if isinstance(val, Enum): + return val + try: + return cls(val) + except ValueError: + values = ", ".join([i.value for i in cls]) + msg = ( + f"{val!r} is not a valid value for {cls.__name__}. Use one of {values} " + "instead." + ) + raise ValueError(msg) from None + + def create_converter() -> ts.Converter: """Create the converter.""" converter = ts.converters.get_default_ts_converter() - converter.scalar_converters[Enum] = ( - lambda val, cls: val if isinstance(val, cls) else cls(val) - ) + converter.scalar_converters[Enum] = _convert_to_enum converter.scalar_converters[PluginManager] = ( lambda val, cls: val if isinstance(val, cls) else cls(**val) ) @@ -200,9 +192,9 @@ def create_settings_loaders(kwargs: dict[str, Any] | None = None) -> list[Loader "*.toml": TomlFormat( section="tool.pytask.ini_options", deprecated=( - "[skipped]Deprecation Warning! Configuring pytask in the " - r"section \[tool.pytask.ini_options] is deprecated and will be " - r"removed in v0.6. Please, use \[tool.pytask] instead.[/]\n\n" + "DeprecationWarning: Configuring pytask in the " + "section \\[tool.pytask.ini_options] is deprecated and will be " + "removed in v0.6. Please, use \\[tool.pytask] instead." ), ) }, @@ -245,12 +237,29 @@ def convert_settings_to_kwargs(settings: Settings) -> dict[str, Any]: def _rewrite_paths_of_options( - settings: SettingsDict, options: OptionList + settings: SettingsDict, options: OptionList, section: str | None ) -> SettingsDict: """Rewrite paths of options in the settings.""" - option_name_to_path = {option.path.split(".")[1]: option.path for option in options} - return { - option_name_to_path[name]: value - for name, value in settings.items() - if name in option_name_to_path + option_paths = {option.path for option in options} + option_name_to_path = { + option.path.rsplit(".", maxsplit=1)[1]: option.path for option in options } + + new_settings = {} + for name, value in settings.items(): + if name in option_paths: + new_settings[name] = value + continue + + if name in option_name_to_path: + new_path = option_name_to_path[name] + if section: + subsection, _ = new_path.rsplit(".", maxsplit=1) + msg = ( + f"DeprecationWarning: The path of the option {name!r} changed from " + f"\\[{section}] to the new path \\[tool.pytask.{subsection}]." + ) + console.print(msg, style="skipped") + new_settings[new_path] = value + + return new_settings diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py index 63ff3800..e7ece320 100644 --- a/src/_pytask/traceback.py +++ b/src/_pytask/traceback.py @@ -19,6 +19,7 @@ import _pytask from _pytask.outcomes import Exit from _pytask.tree_util import TREE_UTIL_LIB_DIRECTORY +import typed_settings as ts if TYPE_CHECKING: from rich.console import Console @@ -35,6 +36,7 @@ _PLUGGY_DIRECTORY = Path(pluggy.__file__).parent _PYTASK_DIRECTORY = Path(_pytask.__file__).parent +_TYPED_SETTINGS_DIRECTORY = Path(ts.__file__).parent ExceptionInfo: TypeAlias = Tuple[ @@ -51,7 +53,8 @@ class Traceback: _show_locals: ClassVar[bool] = False suppress: ClassVar[tuple[Path, ...]] = ( _PLUGGY_DIRECTORY, - # _PYTASK_DIRECTORY, + _PYTASK_DIRECTORY, + _TYPED_SETTINGS_DIRECTORY, TREE_UTIL_LIB_DIRECTORY, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 01b40c84..66af9043 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -34,14 +34,21 @@ def test_help_pages(runner, commands, help_option): @pytest.mark.end_to_end() -def test_help_texts_are_modified_by_config(runner, tmp_path): +@pytest.mark.parametrize("config_section", ["pytask.ini_options", "pytask"]) +def test_help_texts_are_modified_by_config(tmp_path, config_section): tmp_path.joinpath("pyproject.toml").write_text( - '[tool.pytask.ini_options]\nshow_capture = "stdout"' + f'[tool.{config_section}]\nshow_capture = "stdout"' ) + result = run_in_subprocess(("pytask", "build", "--help"), cwd=tmp_path) + assert "[default:" in result.stdout + assert " stdout]" in result.stdout - result = runner.invoke( - cli, - ["build", "--help", "--config", tmp_path.joinpath("pyproject.toml").as_posix()], - ) - assert "[default: stdout]" in result.output +def test_precendence_of_new_to_old_section(tmp_path): + tmp_path.joinpath("pyproject.toml").write_text( + '[tool.pytask.ini_options]\nshow_capture = "stdout"\n\n' + '[tool.pytask]\nshow_capture = "stderr"' + ) + result = run_in_subprocess(("pytask", "build", "--help"), cwd=tmp_path) + assert "[default:" in result.stdout + assert " stderr]" in result.stdout diff --git a/tests/test_config.py b/tests/test_config.py index 5f1f55cc..92484e6d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -123,3 +123,15 @@ def test_paths_are_relative_to_configuration_file(tmp_path): result = run_in_subprocess(("python", "script.py"), cwd=tmp_path) assert result.exit_code == ExitCode.OK assert "1 Succeeded" in result.stdout + + +def test_old_config_section_is_deprecated(): ... + + +def test_new_config_section_is_not_deprecated(): ... + + +def test_old_config_path_is_deprecated(): ... + + +def test_new_config_path_is_not_deprecated(): ... From 78e4ebc66fd922c899e4784c3b249b848dcd9465 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 9 May 2024 00:59:17 +0200 Subject: [PATCH 13/15] fix some issues. --- .pre-commit-config.yaml | 1 + pyproject.toml | 4 ---- src/_pytask/dag_command.py | 4 ++-- src/_pytask/settings_utils.py | 10 ++++++---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 16b2035a..95cafe99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,7 @@ repos: rev: v2.0.0 hooks: - id: refurb + additional_dependencies: [typed-settings] - repo: https://github.com/executablebooks/mdformat rev: 0.7.17 hooks: diff --git a/pyproject.toml b/pyproject.toml index a3569330..80c2b9c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -195,10 +195,6 @@ ignore_errors = true module = ["click_default_group", "networkx"] ignore_missing_imports = true -[[tool.mypy.overrides]] -module = ["_pytask.coiled_utils"] -disable_error_code = ["import-not-found"] - [[tool.mypy.overrides]] module = ["_pytask.hookspecs"] disable_error_code = ["empty-body"] diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index 2d336c93..0defbdfa 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -147,13 +147,13 @@ def build_dag( } if settings is None: - from _pytask.cli import settings_builders + from _pytask.cli import settings_builder pm = get_plugin_manager() storage.store(pm) settings = ts.load_settings( - settings_builders["dag"].build_settings(), create_settings_loaders() + settings_builder["dag"].build_settings(), create_settings_loaders() ) else: pm = storage.get() diff --git a/src/_pytask/settings_utils.py b/src/_pytask/settings_utils.py index 18c6241d..f30299d3 100644 --- a/src/_pytask/settings_utils.py +++ b/src/_pytask/settings_utils.py @@ -41,7 +41,7 @@ def _handle_enum(type: type[Enum], default: Any, is_optional: bool) -> dict[str, Any]: # noqa: A002 """Use enum values as choices for click options.""" - kwargs = {"type": click.Choice([i.value for i in type])} + kwargs: dict[str, Any] = {"type": click.Choice([i.value for i in type])} if isinstance(default, type): kwargs["default"] = default.value elif is_optional: @@ -134,7 +134,7 @@ def __call__( class DictLoader: """Load settings from a dict of values.""" - def __init__(self, settings: dict) -> None: + def __init__(self, settings: dict[str, Any]) -> None: self.settings = settings def __call__( @@ -143,7 +143,9 @@ def __call__( options: OptionList, ) -> LoadedSettings: settings = _rewrite_paths_of_options(self.settings, options, section=None) - nested_settings = {name.split(".")[0]: {} for name in settings} + nested_settings: dict[str, dict[str, Any]] = { + name.split(".")[0]: {} for name in settings + } for long_name, value in settings.items(): group, name = long_name.split(".") nested_settings[group][name] = value @@ -229,7 +231,7 @@ def update_settings(settings: Any, updates: dict[str, Any]) -> Any: def convert_settings_to_kwargs(settings: Settings) -> dict[str, Any]: """Convert the settings to kwargs.""" - kwargs = {} + kwargs: dict[str, Any] = {} names = [i for i in dir(settings) if not i.startswith("_")] for name in names: kwargs = kwargs | attrs.asdict(getattr(settings, name)) From a61cd7dbe8234abab1f6052517989574bb4a381a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 May 2024 22:59:50 +0000 Subject: [PATCH 14/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/_pytask/cli.py | 4 ++-- src/_pytask/parameters.py | 3 ++- src/_pytask/traceback.py | 2 +- tests/test_collect_utils.py | 6 +++++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index 7561c6c2..30eb5d2d 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -5,15 +5,15 @@ import sys from typing import Any -from _pytask.traceback import Traceback import click from packaging.version import parse as parse_version from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup +from _pytask.console import console from _pytask.pluginmanager import storage from _pytask.settings_utils import SettingsBuilder -from _pytask.console import console +from _pytask.traceback import Traceback _CONTEXT_SETTINGS: dict[str, Any] = { "help_option_names": ("-h", "--help"), diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index b5d2848d..5d7009aa 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -11,7 +11,6 @@ import click import typed_settings as ts from click import Context -from pluggy import PluginManager from _pytask.path import import_path from _pytask.pluginmanager import get_plugin_manager @@ -20,6 +19,8 @@ from _pytask.pluginmanager import storage if TYPE_CHECKING: + from pluggy import PluginManager + from _pytask.settings_utils import SettingsBuilder diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py index e7ece320..2d373b3f 100644 --- a/src/_pytask/traceback.py +++ b/src/_pytask/traceback.py @@ -12,6 +12,7 @@ from typing import Union import pluggy +import typed_settings as ts from attrs import define from attrs import field from rich.traceback import Traceback as RichTraceback @@ -19,7 +20,6 @@ import _pytask from _pytask.outcomes import Exit from _pytask.tree_util import TREE_UTIL_LIB_DIRECTORY -import typed_settings as ts if TYPE_CHECKING: from rich.console import Console diff --git a/tests/test_collect_utils.py b/tests/test_collect_utils.py index b23674ac..ab5dbf5d 100644 --- a/tests/test_collect_utils.py +++ b/tests/test_collect_utils.py @@ -1,10 +1,14 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest from _pytask.collect_utils import _find_args_with_product_annotation -from pytask import Product from typing_extensions import Annotated +if TYPE_CHECKING: + from pytask import Product + @pytest.mark.unit() def test_find_args_with_product_annotation(): From f74f0a3ffb1e00c5ab394e4bdf027e42256ed7fe Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 14 May 2024 23:45:42 +0200 Subject: [PATCH 15/15] Filter also context. --- src/_pytask/cli.py | 4 ++-- src/_pytask/parameters.py | 2 +- src/_pytask/traceback.py | 26 ++++++++++++++------------ tests/test_collect_utils.py | 6 +++++- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index 7561c6c2..30eb5d2d 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -5,15 +5,15 @@ import sys from typing import Any -from _pytask.traceback import Traceback import click from packaging.version import parse as parse_version from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup +from _pytask.console import console from _pytask.pluginmanager import storage from _pytask.settings_utils import SettingsBuilder -from _pytask.console import console +from _pytask.traceback import Traceback _CONTEXT_SETTINGS: dict[str, Any] = { "help_option_names": ("-h", "--help"), diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index b5d2848d..984ba396 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -11,7 +11,7 @@ import click import typed_settings as ts from click import Context -from pluggy import PluginManager +from pluggy import PluginManager # noqa: TCH002 from _pytask.path import import_path from _pytask.pluginmanager import get_plugin_manager diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py index e7ece320..681aed1e 100644 --- a/src/_pytask/traceback.py +++ b/src/_pytask/traceback.py @@ -12,6 +12,7 @@ from typing import Union import pluggy +import typed_settings as ts from attrs import define from attrs import field from rich.traceback import Traceback as RichTraceback @@ -19,7 +20,6 @@ import _pytask from _pytask.outcomes import Exit from _pytask.tree_util import TREE_UTIL_LIB_DIRECTORY -import typed_settings as ts if TYPE_CHECKING: from rich.console import Console @@ -38,6 +38,13 @@ _PYTASK_DIRECTORY = Path(_pytask.__file__).parent _TYPED_SETTINGS_DIRECTORY = Path(ts.__file__).parent +_DEFAULT_SUPPRESS = ( + _PLUGGY_DIRECTORY, + _PYTASK_DIRECTORY, + _TYPED_SETTINGS_DIRECTORY, + TREE_UTIL_LIB_DIRECTORY, +) + ExceptionInfo: TypeAlias = Tuple[ Type[BaseException], BaseException, Union[TracebackType, None] @@ -51,12 +58,7 @@ class Traceback: show_locals: bool = field() _show_locals: ClassVar[bool] = False - suppress: ClassVar[tuple[Path, ...]] = ( - _PLUGGY_DIRECTORY, - _PYTASK_DIRECTORY, - _TYPED_SETTINGS_DIRECTORY, - TREE_UTIL_LIB_DIRECTORY, - ) + suppress: ClassVar[tuple[Path, ...]] = _DEFAULT_SUPPRESS @show_locals.default def _show_locals_default(self) -> bool: @@ -93,6 +95,7 @@ def _remove_internal_traceback_frames_from_exc_info( suppress: tuple[Path, ...] = ( _PLUGGY_DIRECTORY, TREE_UTIL_LIB_DIRECTORY, + _TYPED_SETTINGS_DIRECTORY, _PYTASK_DIRECTORY, ), ) -> OptionalExceptionInfo: @@ -106,6 +109,9 @@ def _remove_internal_traceback_frames_from_exc_info( exc_info[1].__cause__ = _remove_internal_traceback_frames_from_exception( exc_info[1].__cause__ ) + exc_info[1].__context__ = _remove_internal_traceback_frames_from_exception( + exc_info[1].__context__ + ) if isinstance(exc_info[2], TracebackType): filtered_traceback = _filter_internal_traceback_frames(exc_info, suppress) @@ -135,11 +141,7 @@ def _remove_internal_traceback_frames_from_exception( def _is_internal_or_hidden_traceback_frame( frame: TracebackType, exc_info: ExceptionInfo, - suppress: tuple[Path, ...] = ( - _PLUGGY_DIRECTORY, - # _PYTASK_DIRECTORY, - TREE_UTIL_LIB_DIRECTORY, - ), + suppress: tuple[Path, ...] = _DEFAULT_SUPPRESS, ) -> bool: """Return ``True`` if traceback frame belongs to internal packages or is hidden. diff --git a/tests/test_collect_utils.py b/tests/test_collect_utils.py index b23674ac..ab5dbf5d 100644 --- a/tests/test_collect_utils.py +++ b/tests/test_collect_utils.py @@ -1,10 +1,14 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest from _pytask.collect_utils import _find_args_with_product_annotation -from pytask import Product from typing_extensions import Annotated +if TYPE_CHECKING: + from pytask import Product + @pytest.mark.unit() def test_find_args_with_product_annotation():