From 81b6b03848b300f2f6f0de0e4e325447cf514317 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Dec 2023 01:32:12 +0100 Subject: [PATCH 1/6] Add error message for not collected tasks with @task decorator. --- docs/source/changes.md | 4 +++- src/_pytask/collect.py | 35 +++++++++++++++++++++++++++++++++++ src/_pytask/task.py | 3 ++- tests/test_task.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index f5202905..612d1763 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -12,7 +12,9 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and (fixes #514). It also warns if the path is configured as a string and not a list of strings. - {pull}`519` raises an error when builtin functions are wrapped with - {func}`~pytask.task`. Closes {issue}`512`. + {func}`~pytask.task`. Closes {issue}`512`.pull +- {pull}`520` raises an error message when imported functions are wrapped with + {func}`@task ` in a task module. Fixes {issue}`513`. ## 0.4.4 - 2023-12-04 diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 786fbf63..3d99d78d 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -22,6 +22,7 @@ from _pytask.console import get_file from _pytask.console import is_jupyter from _pytask.exceptions import CollectionError +from _pytask.exceptions import NodeNotCollectedError from _pytask.mark_utils import get_all_marks from _pytask.mark_utils import has_mark from _pytask.node_protocols import PNode @@ -38,6 +39,7 @@ from _pytask.path import shorten_path from _pytask.reports import CollectionReport from _pytask.shared import find_duplicates +from _pytask.task_utils import COLLECTED_TASKS from _pytask.task_utils import task as task_decorator from _pytask.typing import is_task_function from rich.text import Text @@ -54,6 +56,7 @@ def pytask_collect(session: Session) -> bool: _collect_from_paths(session) _collect_from_tasks(session) + _collect_not_collected_tasks(session) session.tasks.extend( i.node @@ -135,6 +138,38 @@ def _collect_from_tasks(session: Session) -> None: session.collection_reports.append(report) +_FAILED_COLLECTING_TASK = """\ +Failed to collect task '{name}' from file '{path}'. + +This can happen when the task function is defined in another module, imported to a \ +task module and wrapped with the '@task' decorator. + +To collect this task correctly, wrap the imported function in a lambda expression like + +task(...)(lambda **x: imported_function(**x)). +""" + + +def _collect_not_collected_tasks(session: Session) -> None: + """Collect tasks that are not collected yet and create failed reports.""" + for path, tasks in COLLECTED_TASKS.items(): + for task in tasks: + name = task.pytask_meta.name # type: ignore[attr-defined] + node = Task(base_name=name, path=path, function=task) + report = CollectionReport( + outcome=CollectionOutcome.FAIL, + node=node, + exc_info=( + NodeNotCollectedError, + NodeNotCollectedError( + _FAILED_COLLECTING_TASK.format(name=name, path=node.path) + ), + None, + ), + ) + session.collection_reports.append(report) + + @hookimpl def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: """Ignore a path during the collection.""" diff --git a/src/_pytask/task.py b/src/_pytask/task.py index 7f898721..7015e298 100644 --- a/src/_pytask/task.py +++ b/src/_pytask/task.py @@ -76,6 +76,7 @@ def _raise_error_when_task_functions_are_duplicated( msg = ( "There are some duplicates among the repeated tasks. It happens when you define" "the task function outside the loop body and merely wrap in the loop body with " - f"the '@task(...)' decorator.\n\n{flat_tree}" + "the 'task(...)(func)' decorator. As a workaround, wrap the task function in " + f"a lambda expression like 'task(...)(lambda **x: func(**x))'.\n\n{flat_tree}" ) raise ValueError(msg) diff --git a/tests/test_task.py b/tests/test_task.py index b95194ec..26a361a3 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -679,3 +679,31 @@ def test_raise_error_with_builtin_function_as_task(runner, tmp_path): result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.COLLECTION_FAILED assert "Builtin functions cannot be wrapped" in result.output + + +def test_task_function_in_another_module(runner, tmp_path): + source = """ + def func(): + return "Hello, World!" + """ + tmp_path.joinpath("module.py").write_text(textwrap.dedent(source)) + + source = """ + from pytask import task + from pathlib import Path + from _pytask.path import import_path + import inspect + + _ROOT_PATH = Path(__file__).parent + + module = import_path(_ROOT_PATH / "module.py", _ROOT_PATH) + name_to_obj = dict(inspect.getmembers(module)) + + task_example = task(produces=Path("out.txt"))(name_to_obj["func"]) + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "1 Succeeded" in result.output + assert tmp_path.joinpath("out.txt").read_text() == "Hello, World!" From cfaa0991bcd563d9124706f9bbefbd0a319444e3 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Dec 2023 01:32:52 +0100 Subject: [PATCH 2/6] Fix. --- docs/source/changes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index 612d1763..2d5cc3c4 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -13,7 +13,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and strings. - {pull}`519` raises an error when builtin functions are wrapped with {func}`~pytask.task`. Closes {issue}`512`.pull -- {pull}`520` raises an error message when imported functions are wrapped with +- {pull}`521` raises an error message when imported functions are wrapped with {func}`@task ` in a task module. Fixes {issue}`513`. ## 0.4.4 - 2023-12-04 From e9b65da45b554e35a98189b289e19b0bb5f780e8 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Dec 2023 19:00:56 +0100 Subject: [PATCH 3/6] fix. --- src/_pytask/collect.py | 3 ++- tests/test_task.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 3d99d78d..be2d7759 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -152,7 +152,8 @@ def _collect_from_tasks(session: Session) -> None: def _collect_not_collected_tasks(session: Session) -> None: """Collect tasks that are not collected yet and create failed reports.""" - for path, tasks in COLLECTED_TASKS.items(): + for path in list(COLLECTED_TASKS): + tasks = COLLECTED_TASKS.pop(path) for task in tasks: name = task.pytask_meta.name # type: ignore[attr-defined] node = Task(base_name=name, path=path, function=task) diff --git a/tests/test_task.py b/tests/test_task.py index 26a361a3..093d39e4 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -704,6 +704,5 @@ def func(): tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert "1 Succeeded" in result.output - assert tmp_path.joinpath("out.txt").read_text() == "Hello, World!" + assert result.exit_code == ExitCode.COLLECTION_FAILED + assert "1 Failed" in result.output From 2a30aed8fb201c954b296505c4a827f672d85f74 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Dec 2023 23:14:24 +0100 Subject: [PATCH 4/6] f --- src/_pytask/console.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/_pytask/console.py b/src/_pytask/console.py index a0bf5b01..e3c2def5 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -225,18 +225,13 @@ def get_file( # noqa: PLR0911 return get_file(function.__wrapped__) source_file = inspect.getsourcefile(function) if source_file: - # Handle functions defined in the REPL. if "" in source_file: return None - # Handle lambda functions. if "" in source_file: try: return Path(function.__globals__["__file__"]).absolute().resolve() except KeyError: return None - # Handle functions defined in Jupyter notebooks. - if "ipykernel" in source_file or "ipython-input" in source_file: - return None return Path(source_file).absolute().resolve() return None From 6ed77f57a560e4f58739f40e89f4069b6f8bccfc Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Dec 2023 23:31:41 +0100 Subject: [PATCH 5/6] Finish. --- src/_pytask/collect.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index d2cfedba..0fae300b 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -103,6 +103,9 @@ def _collect_from_tasks(session: Session) -> None: path = get_file(raw_task) name = raw_task.pytask_meta.name + if has_mark(raw_task, "task"): + COLLECTED_TASKS[path].remove(raw_task) + # When a task is not a callable, it can be anything or a PTask. Set arbitrary # values and it will pass without errors and not collected. else: From 3c3c744feb1cb84d70aa6b93aea0fa9242c72312 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 9 Dec 2023 10:21:44 +0100 Subject: [PATCH 6/6] FIx. --- tests/test_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_task.py b/tests/test_task.py index 093d39e4..83357627 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -699,7 +699,7 @@ def func(): module = import_path(_ROOT_PATH / "module.py", _ROOT_PATH) name_to_obj = dict(inspect.getmembers(module)) - task_example = task(produces=Path("out.txt"))(name_to_obj["func"]) + task(produces=Path("out.txt"))(name_to_obj["func"]) """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))