From 2cd96d761a05ad2fba7fa3bb78bc9bb631844208 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 21 Oct 2023 17:28:31 +0200
Subject: [PATCH 01/81] Simplify TopologicalSorter.
---
src/_pytask/dag_utils.py | 28 ++++++++--------------------
src/_pytask/execute.py | 4 +---
tests/test_dag_utils.py | 27 +--------------------------
3 files changed, 10 insertions(+), 49 deletions(-)
diff --git a/src/_pytask/dag_utils.py b/src/_pytask/dag_utils.py
index 650cf75c..3c74c7fd 100644
--- a/src/_pytask/dag_utils.py
+++ b/src/_pytask/dag_utils.py
@@ -69,15 +69,12 @@ class TopologicalSorter:
dag: nx.DiGraph
dag_backup: nx.DiGraph
priorities: dict[str, int] = field(factory=dict)
- _is_prepared: bool = False
_nodes_out: set[str] = field(factory=set)
@classmethod
def from_dag(cls, dag: nx.DiGraph) -> TopologicalSorter:
"""Instantiate from a DAG."""
- if not dag.is_directed():
- msg = "Only directed graphs have a topological order."
- raise ValueError(msg)
+ cls.check_dag(dag)
tasks = [
dag.nodes[node]["task"] for node in dag.nodes if "task" in dag.nodes[node]
@@ -90,23 +87,22 @@ def from_dag(cls, dag: nx.DiGraph) -> TopologicalSorter:
return cls(dag=task_dag, priorities=priorities, dag_backup=task_dag.copy())
- def prepare(self) -> None:
- """Perform some checks before creating a topological ordering."""
+ @staticmethod
+ def check_dag(dag: nx.DiGraph) -> None:
+ if not dag.is_directed():
+ msg = "Only directed graphs have a topological order."
+ raise ValueError(msg)
+
try:
- nx.algorithms.cycles.find_cycle(self.dag)
+ nx.algorithms.cycles.find_cycle(dag)
except nx.NetworkXNoCycle:
pass
else:
msg = "The DAG contains cycles."
raise ValueError(msg)
- self._is_prepared = True
-
def get_ready(self, n: int = 1) -> list[str]:
"""Get up to ``n`` tasks which are ready."""
- if not self._is_prepared:
- msg = "The TopologicalSorter needs to be prepared."
- raise ValueError(msg)
if not isinstance(n, int) or n < 1:
msg = "'n' must be an integer greater or equal than 1."
raise ValueError(msg)
@@ -129,16 +125,8 @@ def done(self, *nodes: str) -> None:
self._nodes_out = self._nodes_out - set(nodes)
self.dag.remove_nodes_from(nodes)
- def reset(self) -> None:
- """Reset an exhausted topological sorter."""
- if self.dag_backup:
- self.dag = self.dag_backup.copy()
- self._is_prepared = False
- self._nodes_out = set()
-
def static_order(self) -> Generator[str, None, None]:
"""Return a topological order of tasks as an iterable."""
- self.prepare()
while self.is_active():
new_task = self.get_ready()[0]
yield new_task
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index e468340f..5ba15aac 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -75,9 +75,7 @@ def pytask_execute_log_start(session: Session) -> None:
@hookimpl(trylast=True)
def pytask_execute_create_scheduler(session: Session) -> TopologicalSorter:
"""Create a scheduler based on topological sorting."""
- scheduler = TopologicalSorter.from_dag(session.dag)
- scheduler.prepare()
- return scheduler
+ return TopologicalSorter.from_dag(session.dag)
@hookimpl
diff --git a/tests/test_dag_utils.py b/tests/test_dag_utils.py
index 3b25a8f0..aa675daa 100644
--- a/tests/test_dag_utils.py
+++ b/tests/test_dag_utils.py
@@ -140,37 +140,12 @@ def test_raise_error_for_undirected_graphs(dag):
@pytest.mark.unit()
def test_raise_error_for_cycle_in_graph(dag):
dag.add_edge(".::4", ".::1")
- scheduler = TopologicalSorter.from_dag(dag)
with pytest.raises(ValueError, match="The DAG contains cycles."):
- scheduler.prepare()
-
-
-@pytest.mark.unit()
-def test_raise_if_topological_sorter_is_not_prepared(dag):
- scheduler = TopologicalSorter.from_dag(dag)
- with pytest.raises(ValueError, match="The TopologicalSorter needs to be prepared."):
- scheduler.get_ready(1)
+ TopologicalSorter.from_dag(dag)
@pytest.mark.unit()
def test_ask_for_invalid_number_of_ready_tasks(dag):
scheduler = TopologicalSorter.from_dag(dag)
- scheduler.prepare()
with pytest.raises(ValueError, match="'n' must be"):
scheduler.get_ready(0)
-
-
-@pytest.mark.unit()
-def test_reset_topological_sorter(dag):
- scheduler = TopologicalSorter.from_dag(dag)
- scheduler.prepare()
- name = scheduler.get_ready()[0]
- scheduler.done(name)
-
- assert scheduler._is_prepared
- assert name not in scheduler.dag.nodes
-
- scheduler.reset()
-
- assert not scheduler._is_prepared
- assert name in scheduler.dag.nodes
From 6f2aaef48a175a3b59d93bedbeaee98f6eedfe43 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 21 Oct 2023 17:50:28 +0200
Subject: [PATCH 02/81] Remove static_order.
---
src/_pytask/dag.py | 4 +++-
src/_pytask/dag_utils.py | 7 -------
src/_pytask/execute.py | 6 ++++--
tests/test_dag_utils.py | 7 ++++++-
4 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index b67ec49e..43d537fe 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -122,7 +122,8 @@ def pytask_dag_select_execution_dag(session: Session, dag: nx.DiGraph) -> None:
scheduler = TopologicalSorter.from_dag(dag)
visited_nodes: set[str] = set()
- for task_name in scheduler.static_order():
+ while scheduler.is_active():
+ task_name = scheduler.get_ready()[0]
if task_name not in visited_nodes:
task = dag.nodes[task_name]["task"]
have_changed = _have_task_or_neighbors_changed(session, dag, task)
@@ -132,6 +133,7 @@ def pytask_dag_select_execution_dag(session: Session, dag: nx.DiGraph) -> None:
dag.nodes[task_name]["task"].markers.append(
Mark("skip_unchanged", (), {})
)
+ scheduler.done(task_name)
@hookimpl
diff --git a/src/_pytask/dag_utils.py b/src/_pytask/dag_utils.py
index 3c74c7fd..3218bfeb 100644
--- a/src/_pytask/dag_utils.py
+++ b/src/_pytask/dag_utils.py
@@ -125,13 +125,6 @@ def done(self, *nodes: str) -> None:
self._nodes_out = self._nodes_out - set(nodes)
self.dag.remove_nodes_from(nodes)
- def static_order(self) -> Generator[str, None, None]:
- """Return a topological order of tasks as an iterable."""
- while self.is_active():
- new_task = self.get_ready()[0]
- yield new_task
- self.done(new_task)
-
def _extract_priorities_from_tasks(tasks: list[PTask]) -> dict[str, int]:
"""Extract priorities from tasks.
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 5ba15aac..b2ad6bf8 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -82,12 +82,14 @@ def pytask_execute_create_scheduler(session: Session) -> TopologicalSorter:
def pytask_execute_build(session: Session) -> bool | None:
"""Execute tasks."""
if isinstance(session.scheduler, TopologicalSorter):
- for name in session.scheduler.static_order():
- task = session.dag.nodes[name]["task"]
+ while session.scheduler.is_active():
+ task_name = session.scheduler.get_ready()[0]
+ task = session.dag.nodes[task_name]["task"]
report = session.hook.pytask_execute_task_protocol(
session=session, task=task
)
session.execution_reports.append(report)
+ session.scheduler.done(task_name)
if session.should_stop:
return True
diff --git a/tests/test_dag_utils.py b/tests/test_dag_utils.py
index aa675daa..8512f394 100644
--- a/tests/test_dag_utils.py
+++ b/tests/test_dag_utils.py
@@ -29,7 +29,12 @@ def dag():
@pytest.mark.unit()
def test_sort_tasks_topologically(dag):
- topo_ordering = list(TopologicalSorter.from_dag(dag).static_order())
+ dag = TopologicalSorter.from_dag(dag)
+ topo_ordering = []
+ while dag.is_active():
+ task_name = dag.get_ready()[0]
+ topo_ordering.append(task_name)
+ dag.done(task_name)
assert topo_ordering == [f".::{i}" for i in range(5)]
From e37abd6bf8a272ae143b9e0894c2f702d54d6f65 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 21 Oct 2023 17:52:33 +0200
Subject: [PATCH 03/81] Remove backup.
---
src/_pytask/dag_utils.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/_pytask/dag_utils.py b/src/_pytask/dag_utils.py
index 3218bfeb..29e5748e 100644
--- a/src/_pytask/dag_utils.py
+++ b/src/_pytask/dag_utils.py
@@ -67,7 +67,6 @@ class TopologicalSorter:
"""
dag: nx.DiGraph
- dag_backup: nx.DiGraph
priorities: dict[str, int] = field(factory=dict)
_nodes_out: set[str] = field(factory=set)
@@ -85,7 +84,7 @@ def from_dag(cls, dag: nx.DiGraph) -> TopologicalSorter:
task_dict = {name: nx.ancestors(dag, name) & task_names for name in task_names}
task_dag = nx.DiGraph(task_dict).reverse()
- return cls(dag=task_dag, priorities=priorities, dag_backup=task_dag.copy())
+ return cls(dag=task_dag, priorities=priorities)
@staticmethod
def check_dag(dag: nx.DiGraph) -> None:
From 789e0d8e438f57e7352373379b822c8d67f300b1 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 22 Oct 2023 14:01:06 +0200
Subject: [PATCH 04/81] Add instantiation from another sorter and a DAG.
---
src/_pytask/dag_utils.py | 32 +++++++++++++++++++++++++++-----
tests/test_dag_utils.py | 19 +++++++++++++++++++
2 files changed, 46 insertions(+), 5 deletions(-)
diff --git a/src/_pytask/dag_utils.py b/src/_pytask/dag_utils.py
index 29e5748e..83bb1464 100644
--- a/src/_pytask/dag_utils.py
+++ b/src/_pytask/dag_utils.py
@@ -62,13 +62,22 @@ def node_and_neighbors(dag: nx.DiGraph, node: str) -> Iterable[str]:
class TopologicalSorter:
"""The topological sorter class.
- This class allows to perform a topological sort
+ This class allows to perform a topological sort#
+
+ Attributes
+ ----------
+ dag
+ Not the full DAG, but a reduced version that only considers tasks.
+ priorities
+ A dictionary of task names to a priority value. 1 for try first, 0 for the
+ default priority and, -1 for try last.
"""
dag: nx.DiGraph
priorities: dict[str, int] = field(factory=dict)
- _nodes_out: set[str] = field(factory=set)
+ _nodes_processing: set[str] = field(factory=set)
+ _nodes_done: set[str] = field(factory=set)
@classmethod
def from_dag(cls, dag: nx.DiGraph) -> TopologicalSorter:
@@ -86,6 +95,16 @@ def from_dag(cls, dag: nx.DiGraph) -> TopologicalSorter:
return cls(dag=task_dag, priorities=priorities)
+ @classmethod
+ def from_dag_and_sorter(
+ cls, dag: nx.DiGraph, sorter: TopologicalSorter
+ ) -> TopologicalSorter:
+ """Instantiate a sorter from another sorter and a DAG."""
+ new_sorter = cls.from_dag(dag)
+ new_sorter.done(*sorter._nodes_done)
+ new_sorter._nodes_processing = sorter._nodes_processing
+ return new_sorter
+
@staticmethod
def check_dag(dag: nx.DiGraph) -> None:
if not dag.is_directed():
@@ -106,12 +125,14 @@ def get_ready(self, n: int = 1) -> list[str]:
msg = "'n' must be an integer greater or equal than 1."
raise ValueError(msg)
- ready_nodes = {v for v, d in self.dag.in_degree() if d == 0} - self._nodes_out
+ ready_nodes = {
+ v for v, d in self.dag.in_degree() if d == 0
+ } - self._nodes_processing
prioritized_nodes = sorted(
ready_nodes, key=lambda x: self.priorities.get(x, 0)
)[-n:]
- self._nodes_out.update(prioritized_nodes)
+ self._nodes_processing.update(prioritized_nodes)
return prioritized_nodes
@@ -121,8 +142,9 @@ def is_active(self) -> bool:
def done(self, *nodes: str) -> None:
"""Mark some tasks as done."""
- self._nodes_out = self._nodes_out - set(nodes)
+ self._nodes_processing = self._nodes_processing - set(nodes)
self.dag.remove_nodes_from(nodes)
+ self._nodes_done.update(nodes)
def _extract_priorities_from_tasks(tasks: list[PTask]) -> dict[str, int]:
diff --git a/tests/test_dag_utils.py b/tests/test_dag_utils.py
index 8512f394..9b840957 100644
--- a/tests/test_dag_utils.py
+++ b/tests/test_dag_utils.py
@@ -16,6 +16,7 @@
@pytest.fixture()
def dag():
+ """Create a dag with five nodes in a line."""
dag = nx.DiGraph()
for i in range(4):
dag.add_node(f".::{i}", task=Task(base_name=str(i), path=Path(), function=None))
@@ -154,3 +155,21 @@ def test_ask_for_invalid_number_of_ready_tasks(dag):
scheduler = TopologicalSorter.from_dag(dag)
with pytest.raises(ValueError, match="'n' must be"):
scheduler.get_ready(0)
+
+
+@pytest.mark.unit()
+def test_instantiate_sorter_from_other_sorter(dag):
+ scheduler = TopologicalSorter.from_dag(dag)
+ for _ in range(2):
+ task_name = scheduler.get_ready()[0]
+ scheduler.done(task_name)
+ assert scheduler._nodes_done == {".::0", ".::1"}
+
+ dag.add_node(".::5", task=Task(base_name="5", path=Path(), function=None))
+ dag.add_edge(".::4", ".::5")
+
+ new_scheduler = TopologicalSorter.from_dag_and_sorter(dag, scheduler)
+ while new_scheduler.is_active():
+ task_name = new_scheduler.get_ready()[0]
+ new_scheduler.done(task_name)
+ assert new_scheduler._nodes_done == {".::0", ".::1", ".::2", ".::3", ".::4", ".::5"}
From b1965848c0d72de79dbcbd6964af5fb7af3507c4 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 23 Oct 2023 09:04:22 +0200
Subject: [PATCH 05/81] Add a delayed node that can serve as a product.
---
src/_pytask/collect.py | 8 ++++++
src/_pytask/dag.py | 4 +++
src/_pytask/execute.py | 8 +++++-
src/_pytask/node_protocols.py | 16 ++++++++++-
src/_pytask/nodes.py | 47 +++++++++++++++++++++++++++++--
src/pytask/__init__.py | 4 +++
tests/test_collect.py | 5 ++++
tests/test_collect_command.py | 53 +++++++++++++++++++++++++++++++++++
tests/test_execute.py | 23 +++++++++++++++
9 files changed, 164 insertions(+), 4 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index c928ecce..e7586f59 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -29,6 +29,7 @@
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
+from _pytask.nodes import DelayedPathNode
from _pytask.nodes import PathNode
from _pytask.nodes import PythonNode
from _pytask.nodes import Task
@@ -329,6 +330,11 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PN
node.name = create_name_of_python_node(node_info)
return node
+ if isinstance(node, DelayedPathNode):
+ if node.root_dir is None:
+ node.root_dir = path
+ node.name = node.root_dir.joinpath(node.pattern).as_posix()
+
if isinstance(node, PPathNode) and not node.path.is_absolute():
node.path = path.joinpath(node.path)
@@ -340,6 +346,8 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PN
)
if isinstance(node, PNode):
+ if not node.name:
+ node.name = create_name_of_python_node(node_info)
return node
if isinstance(node, Path):
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index 43d537fe..fa981555 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -26,6 +26,7 @@
from _pytask.mark_utils import get_marks
from _pytask.mark_utils import has_mark
from _pytask.node_protocols import MetaNode
+from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
@@ -90,6 +91,9 @@ def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
def _add_product(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
"""Add a product to the DAG."""
+ if isinstance(node, PDelayedNode):
+ return
+
dag.add_node(node.name, node=node)
dag.add_edge(task.name, node.name)
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index b2ad6bf8..aabc7921 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -24,6 +24,7 @@
from _pytask.exceptions import NodeNotFoundError
from _pytask.mark import Mark
from _pytask.mark_utils import has_mark
+from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
@@ -195,7 +196,12 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
@hookimpl
def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
- """Check if :class:`_pytask.nodes.PathNode` are produced by a task."""
+ """Check if nodes are produced by a task."""
+ # Replace delayed nodes with their actually resolved nodes.
+ task.produces = tree_map( # type: ignore[assignment]
+ lambda x: x.collect() if isinstance(x, PDelayedNode) else x, task.produces
+ )
+
missing_nodes = []
for product in session.dag.successors(task.name):
node = session.dag.nodes[product]["node"]
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index 3ab328ea..0a098cb9 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -14,7 +14,7 @@
from _pytask.mark import Mark
-__all__ = ["MetaNode", "PNode", "PPathNode", "PTask", "PTaskWithPath"]
+__all__ = ["MetaNode", "PDelayedNode", "PNode", "PPathNode", "PTask", "PTaskWithPath"]
@runtime_checkable
@@ -85,3 +85,17 @@ class PTaskWithPath(PTask, Protocol):
"""
path: Path
+
+
+@runtime_checkable
+class PDelayedNode(Protocol):
+ """A protocol for delayed nodes.
+
+ Delayed nodes are nodes that define how nodes look like instead of the actual nodes.
+ Situations like this can happen if tasks produce an unknown amount of nodes, but the
+ style is known.
+
+ """
+
+ def collect(self) -> list[Any]:
+ """Collect the objects that are defined by the fuzzy node."""
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index e7e34380..940aecb6 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -4,7 +4,7 @@
import functools
import hashlib
import inspect
-from pathlib import Path # noqa: TCH003
+from pathlib import Path
from typing import Any
from typing import Callable
from typing import TYPE_CHECKING
@@ -24,7 +24,7 @@
from _pytask.mark import Mark
-__all__ = ["PathNode", "PythonNode", "Task", "TaskWithoutPath"]
+__all__ = ["DelayedPathNode", "PathNode", "PythonNode", "Task", "TaskWithoutPath"]
@define(kw_only=True)
@@ -245,3 +245,46 @@ def state(self) -> str | None:
return str(hashlib.sha256(value).hexdigest())
return str(hash(value))
return "0"
+
+
+@define(kw_only=True)
+class DelayedPathNode(PNode):
+ """A class for delayed :class:`PathNode`s.
+
+ Attributes
+ ----------
+ root_dir
+ The pattern is interpreted relative to the path given by ``root_dir``. If
+ ``root_dir = None``, it is the directory where the path is defined.
+ pattern
+ Patterns are the same as for :mod:`fnmatch`, with the addition of ``**`` which
+ means "this directory and all subdirectories, recursively".
+ name
+ The name of the node.
+
+ .. warning::
+
+ The node inherits from PNode although it does not provide a state, load, and
+ save method.
+
+ """
+
+ root_dir: Path | None = None
+ pattern: str
+ name: str = ""
+
+ def load(self) -> None:
+ raise NotImplementedError
+
+ def save(self, value: Any) -> None:
+ raise NotImplementedError
+
+ def state(self) -> None:
+ return None
+
+ def collect(self) -> list[Path]:
+ """Collect paths defined by the pattern."""
+ if not isinstance(self.root_dir, Path):
+ msg = f"'root_dir' should be a 'pathlib.Path', but it is {self.root_dir!r}"
+ raise TypeError(msg)
+ return list(self.root_dir.glob(self.pattern))
diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py
index c9c0d256..df503346 100644
--- a/src/pytask/__init__.py
+++ b/src/pytask/__init__.py
@@ -38,10 +38,12 @@
from _pytask.models import CollectionMetadata
from _pytask.models import NodeInfo
from _pytask.node_protocols import MetaNode
+from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
+from _pytask.nodes import DelayedPathNode
from _pytask.nodes import PathNode
from _pytask.nodes import PythonNode
from _pytask.nodes import Task
@@ -87,6 +89,7 @@
"ConfigurationError",
"DagReport",
"DatabaseSession",
+ "DelayedPathNode",
"EnumChoice",
"ExecutionError",
"ExecutionReport",
@@ -100,6 +103,7 @@
"NodeInfo",
"NodeNotCollectedError",
"NodeNotFoundError",
+ "PDelayedNode",
"PathNode",
"Persisted",
"PNode",
diff --git a/tests/test_collect.py b/tests/test_collect.py
index 8a952644..7df381bb 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -547,3 +547,8 @@ def task_example(
result = runner.invoke(cli, [tmp_path.as_posix()])
assert result.exit_code == ExitCode.OK
assert tmp_path.joinpath("subfolder", "out.txt").exists()
+
+
+@pytest.mark.end_to_end()
+def test_task_missing_is_ready_cannot_depend_on_delayed_node():
+ raise NotImplementedError
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 9e8debd3..d3ff2c85 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -633,3 +633,56 @@ def task_example() -> Annotated[Dict[str, str], nodes]:
assert "return::1-0" in output
assert "return::1-1" in output
assert "return::2" in output
+
+
+@pytest.mark.end_to_end()
+def test_collect_task_with_delayed_path_node(runner, tmp_path):
+ source = """
+ from pytask import DelayedPathNode
+ from typing_extensions import Annotated
+
+ def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]: ...
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ result = runner.invoke(cli, ["collect", tmp_path.as_posix()])
+
+ assert result.exit_code == ExitCode.OK
+ captured = result.output.replace("\n", "").replace(" ", "")
+ assert "<Module" in captured
+ assert "task_module.py>" in captured
+ assert "<Function" in captured
+ assert "task_example>" in captured
+
+ result = runner.invoke(cli, ["collect", tmp_path.as_posix(), "--nodes"])
+
+ assert result.exit_code == ExitCode.OK
+ captured = result.output.replace("\n", "").replace(" ", "")
+ assert "<Module" in captured
+ assert "task_module.py>" in captured
+ assert "<Function" in captured
+ assert "task_example>" in captured
+ assert "<Product" in captured
+ assert "/*.txt>" in captured
+
+
+@pytest.mark.end_to_end()
+def test_collect_custom_node_receives_default_name(runner, tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from attrs import define
+
+ @define
+ class CustomNode:
+ name: str = ""
+
+ def state(): return None
+
+
+ def task_example() -> Annotated[None, CustomNode()]: ...
+ """
+ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
+ result = runner.invoke(cli, ["collect", "--nodes", tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ output = result.output.replace(" ", "").replace("\n", "")
+ assert "task_example::return" in output
diff --git a/tests/test_execute.py b/tests/test_execute.py
index bbfea3cf..b7e04c63 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -833,3 +833,26 @@ def task_example(
assert "task_example.py::task_example" in result.output
assert "Exception while loading node" in result.output
assert "_pytask/execute.py" not in result.output
+
+
+@pytest.mark.end_to_end()
+def test_task_with_delayed_path_node(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode
+ from pathlib import Path
+
+
+ def task_example(
+ path = Path(__file__).parent
+ ) -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
+ path.joinpath("a.txt").touch()
+ path.joinpath("b.txt").touch()
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ session = build(paths=tmp_path)
+
+ assert session.exit_code == ExitCode.OK
+ assert len(session.tasks) == 1
+ assert len(session.tasks[0].produces["return"]) == 2
From 7bbb5b8cf1de78778ff18ff3d8207052c99a007c Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 23 Oct 2023 09:54:53 +0200
Subject: [PATCH 06/81] Raise errors when not delayed tasks receive delayed
dependencies.
---
src/_pytask/collect.py | 23 +++++++++++++++++------
src/_pytask/collect_utils.py | 11 +++++++++++
src/_pytask/models.py | 6 +++++-
src/_pytask/report.py | 1 +
src/_pytask/task_utils.py | 16 +++++++++++-----
tests/test_collect.py | 15 +++++++++++++--
tests/test_task_utils.py | 16 ++++++++++++++++
7 files changed, 74 insertions(+), 14 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index e7586f59..e47c755a 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -26,6 +26,7 @@
from _pytask.exceptions import CollectionError
from _pytask.mark import MarkGenerator
from _pytask.mark_utils import has_mark
+from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
@@ -306,7 +307,9 @@ def pytask_collect_task(
@hookimpl(trylast=True)
-def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PNode:
+def pytask_collect_node( # noqa: C901
+ session: Session, path: Path, node_info: NodeInfo
+) -> PNode:
"""Collect a node of a task as a :class:`pytask.PNode`.
Strings are assumed to be paths. This might be a strict assumption, but since this
@@ -326,15 +329,23 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PN
"""
node = node_info.value
+ if isinstance(node, PDelayedNode):
+ if not node_info.is_delayed:
+ msg = (
+ "Only a delayed task can depend on a delayed node. The delayed "
+ f"dependency is {node!r}."
+ )
+ raise ValueError(msg)
+
+ if isinstance(node, DelayedPathNode):
+ if node.root_dir is None:
+ node.root_dir = path
+ node.name = node.root_dir.joinpath(node.pattern).as_posix()
+
if isinstance(node, PythonNode):
node.name = create_name_of_python_node(node_info)
return node
- if isinstance(node, DelayedPathNode):
- if node.root_dir is None:
- node.root_dir = path
- node.name = node.root_dir.joinpath(node.pattern).as_posix()
-
if isinstance(node, PPathNode) and not node.path.is_absolute():
node.path = path.joinpath(node.path)
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index 9e2cf1bb..8ca7d3d1 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -93,6 +93,7 @@ def parse_nodes( # noqa: PLR0913
task_name,
NodeInfo(
arg_name=arg_name,
+ is_delayed=False,
path=(),
value=x,
task_path=task_path,
@@ -256,6 +257,9 @@ def parse_dependencies_from_task_function(
dependencies["depends_on"] = nodes
task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {}
+ is_delayed = (
+ obj.pytask_meta.is_ready is not None if hasattr(obj, "pytask_meta") else False
+ )
signature_defaults = parse_keyword_arguments_from_signature_defaults(obj)
kwargs = {**signature_defaults, **task_kwargs}
kwargs.pop("produces", None)
@@ -270,6 +274,7 @@ def parse_dependencies_from_task_function(
task_name,
NodeInfo(
arg_name="depends_on",
+ is_delayed=is_delayed,
path=(),
value=x,
task_path=task_path,
@@ -314,6 +319,7 @@ def parse_dependencies_from_task_function(
task_name,
NodeInfo(
arg_name=parameter_name, # noqa: B023
+ is_delayed=is_delayed,
path=p,
value=x,
task_path=task_path,
@@ -332,6 +338,7 @@ def parse_dependencies_from_task_function(
node_name = create_name_of_python_node(
NodeInfo(
arg_name=parameter_name,
+ is_delayed=is_delayed,
path=(),
value=value,
task_path=task_path,
@@ -421,6 +428,7 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="produces",
+ is_delayed=False,
path=p,
value=x,
task_path=task_path,
@@ -464,6 +472,7 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name=parameter_name, # noqa: B023
+ is_delayed=False,
path=p,
value=x,
task_path=task_path,
@@ -484,6 +493,7 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="return",
+ is_delayed=False,
path=p,
value=x,
task_path=task_path,
@@ -505,6 +515,7 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="return",
+ is_delayed=False,
path=p,
value=x,
task_path=task_path,
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index 247084d4..dd3e43b8 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -2,6 +2,7 @@
from __future__ import annotations
from typing import Any
+from typing import Callable
from typing import NamedTuple
from typing import TYPE_CHECKING
@@ -20,6 +21,8 @@ class CollectionMetadata:
id_: str | None = None
"""The id for a single parametrization."""
+ is_ready: Callable[..., bool] | None = None
+ """A callable that indicates whether a delayed task is ready."""
kwargs: dict[str, Any] = field(factory=dict)
"""Contains kwargs which are necessary for the task function on execution."""
markers: list[Mark] = field(factory=list)
@@ -32,7 +35,8 @@ class CollectionMetadata:
class NodeInfo(NamedTuple):
arg_name: str
+ is_delayed: bool
path: tuple[str | int, ...]
- value: Any
task_path: Path | None
task_name: str
+ value: Any
diff --git a/src/_pytask/report.py b/src/_pytask/report.py
index 261464e3..62fcbbdb 100644
--- a/src/_pytask/report.py
+++ b/src/_pytask/report.py
@@ -31,6 +31,7 @@ def from_exception(
exc_info: OptionalExceptionInfo,
node: MetaNode | None = None,
) -> CollectionReport:
+ exc_info = remove_internal_traceback_frames_from_exc_info(exc_info)
return cls(outcome=outcome, node=node, exc_info=exc_info)
diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py
index 5e80a187..bd8a6bce 100644
--- a/src/_pytask/task_utils.py
+++ b/src/_pytask/task_utils.py
@@ -40,6 +40,7 @@ def task(
name: str | None = None,
*,
id: str | None = None, # noqa: A002
+ is_ready: Callable[..., bool] | None = None,
kwargs: dict[Any, Any] | None = None,
produces: PyTree[Any] | None = None,
) -> Callable[..., Callable[..., Any]]:
@@ -60,6 +61,9 @@ def task(
id will be generated. See
:doc:`this tutorial <../tutorials/repeating_tasks_with_different_inputs>` for
more information.
+ is_ready
+ A callable that indicates when a delayed task is ready. The value is ``None``
+ for a normal task.
kwargs
A dictionary containing keyword arguments which are passed to the task when it
is executed.
@@ -77,7 +81,7 @@ def task(
from typing_extensions import Annotated
from pytask import task
- @task
+ @task()
def create_text_file() -> Annotated[str, Path("file.txt")]:
return "Hello, World!"
@@ -104,17 +108,19 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
parsed_name = name if isinstance(name, str) else func.__name__
if hasattr(unwrapped, "pytask_meta"):
- unwrapped.pytask_meta.name = parsed_name
+ unwrapped.pytask_meta.id_ = id
+ unwrapped.pytask_meta.is_ready = is_ready
unwrapped.pytask_meta.kwargs = parsed_kwargs
unwrapped.pytask_meta.markers.append(Mark("task", (), {}))
- unwrapped.pytask_meta.id_ = id
+ unwrapped.pytask_meta.name = parsed_name
unwrapped.pytask_meta.produces = produces
else:
unwrapped.pytask_meta = CollectionMetadata(
- name=parsed_name,
+ id_=id,
+ is_ready=is_ready,
kwargs=parsed_kwargs,
markers=[Mark("task", (), {})],
- id_=id,
+ name=parsed_name,
produces=produces,
)
diff --git a/tests/test_collect.py b/tests/test_collect.py
index 7df381bb..d4994d13 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -156,6 +156,7 @@ def test_collect_files_w_custom_file_name_pattern(
Path(),
NodeInfo(
arg_name="",
+ is_delayed=False,
path=(),
value=Path.cwd() / "text.txt",
task_path=Path.cwd() / "task_example.py",
@@ -169,6 +170,7 @@ def test_collect_files_w_custom_file_name_pattern(
Path(),
NodeInfo(
arg_name="",
+ is_delayed=False,
path=(),
value=1,
task_path=Path.cwd() / "task_example.py",
@@ -550,5 +552,14 @@ def task_example(
@pytest.mark.end_to_end()
-def test_task_missing_is_ready_cannot_depend_on_delayed_node():
- raise NotImplementedError
+def test_task_missing_is_ready_cannot_depend_on_delayed_node(runner, tmp_path):
+ source = """
+ from pytask import DelayedPathNode
+
+ def task_example(a = DelayedPathNode(pattern="*.txt")): ...
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.COLLECTION_FAILED
+ assert "Only a delayed task can depend on a delayed dependency." in result.output
diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py
index 36b2b208..2f9d354f 100644
--- a/tests/test_task_utils.py
+++ b/tests/test_task_utils.py
@@ -7,6 +7,8 @@
from _pytask.task_utils import _arg_value_to_id_component
from _pytask.task_utils import _parse_task_kwargs
from attrs import define
+from pytask import Mark
+from pytask import task
@pytest.mark.unit()
@@ -56,3 +58,17 @@ def test_parse_task_kwargs(kwargs, expectation, expected):
with expectation:
result = _parse_task_kwargs(kwargs)
assert result == expected
+
+
+@pytest.mark.integration()
+def test_default_values_of_pytask_meta():
+ @task()
+ def task_example():
+ ...
+
+ assert task_example.pytask_meta.id_ is None
+ assert task_example.pytask_meta.is_ready is None
+ assert task_example.pytask_meta.kwargs == {}
+ assert task_example.pytask_meta.markers == [Mark("task", (), {})]
+ assert task_example.pytask_meta.name == "task_example"
+ assert task_example.pytask_meta.produces is None
From af7eb2c105cbc422871c5023cd34e8ba0689f8a0 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 23 Oct 2023 10:38:05 +0200
Subject: [PATCH 07/81] Everything works except tasks depending on delayed
nodes.
---
src/_pytask/collect.py | 2 +-
src/_pytask/collect_utils.py | 18 +++++++++---------
src/_pytask/execute.py | 3 ++-
src/_pytask/models.py | 2 +-
tests/test_collect.py | 4 ++--
tests/test_execute.py | 33 ++++++++++++++++++++++++++++++++-
6 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index e47c755a..0d905531 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -330,7 +330,7 @@ def pytask_collect_node( # noqa: C901
node = node_info.value
if isinstance(node, PDelayedNode):
- if not node_info.is_delayed:
+ if not node_info.allow_delayed:
msg = (
"Only a delayed task can depend on a delayed node. The delayed "
f"dependency is {node!r}."
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index 8ca7d3d1..55b53f8b 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -93,7 +93,7 @@ def parse_nodes( # noqa: PLR0913
task_name,
NodeInfo(
arg_name=arg_name,
- is_delayed=False,
+ allow_delayed=False,
path=(),
value=x,
task_path=task_path,
@@ -257,7 +257,7 @@ def parse_dependencies_from_task_function(
dependencies["depends_on"] = nodes
task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {}
- is_delayed = (
+ allow_delayed = (
obj.pytask_meta.is_ready is not None if hasattr(obj, "pytask_meta") else False
)
signature_defaults = parse_keyword_arguments_from_signature_defaults(obj)
@@ -274,7 +274,7 @@ def parse_dependencies_from_task_function(
task_name,
NodeInfo(
arg_name="depends_on",
- is_delayed=is_delayed,
+ allow_delayed=allow_delayed,
path=(),
value=x,
task_path=task_path,
@@ -319,7 +319,7 @@ def parse_dependencies_from_task_function(
task_name,
NodeInfo(
arg_name=parameter_name, # noqa: B023
- is_delayed=is_delayed,
+ allow_delayed=allow_delayed,
path=p,
value=x,
task_path=task_path,
@@ -338,7 +338,7 @@ def parse_dependencies_from_task_function(
node_name = create_name_of_python_node(
NodeInfo(
arg_name=parameter_name,
- is_delayed=is_delayed,
+ allow_delayed=allow_delayed,
path=(),
value=value,
task_path=task_path,
@@ -428,7 +428,7 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="produces",
- is_delayed=False,
+ allow_delayed=True,
path=p,
value=x,
task_path=task_path,
@@ -472,7 +472,7 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name=parameter_name, # noqa: B023
- is_delayed=False,
+ allow_delayed=False,
path=p,
value=x,
task_path=task_path,
@@ -493,7 +493,7 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="return",
- is_delayed=False,
+ allow_delayed=True,
path=p,
value=x,
task_path=task_path,
@@ -515,7 +515,7 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="return",
- is_delayed=False,
+ allow_delayed=True,
path=p,
value=x,
task_path=task_path,
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index aabc7921..b58ca471 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -189,7 +189,8 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
nodes = tree_leaves(task.produces["return"])
values = structure_return.flatten_up_to(out)
for node, value in zip(nodes, values):
- node.save(value) # type: ignore[attr-defined]
+ if not isinstance(node, PDelayedNode):
+ node.save(value) # type: ignore[attr-defined]
return True
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index dd3e43b8..2c2cebea 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -35,7 +35,7 @@ class CollectionMetadata:
class NodeInfo(NamedTuple):
arg_name: str
- is_delayed: bool
+ allow_delayed: bool
path: tuple[str | int, ...]
task_path: Path | None
task_name: str
diff --git a/tests/test_collect.py b/tests/test_collect.py
index d4994d13..7f9c1c28 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -156,7 +156,7 @@ def test_collect_files_w_custom_file_name_pattern(
Path(),
NodeInfo(
arg_name="",
- is_delayed=False,
+ allow_delayed=False,
path=(),
value=Path.cwd() / "text.txt",
task_path=Path.cwd() / "task_example.py",
@@ -170,7 +170,7 @@ def test_collect_files_w_custom_file_name_pattern(
Path(),
NodeInfo(
arg_name="",
- is_delayed=False,
+ allow_delayed=False,
path=(),
value=1,
task_path=Path.cwd() / "task_example.py",
diff --git a/tests/test_execute.py b/tests/test_execute.py
index b7e04c63..0c3642bc 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -836,7 +836,7 @@ def task_example(
@pytest.mark.end_to_end()
-def test_task_with_delayed_path_node(tmp_path):
+def test_task_that_produces_delayed_path_node(tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DelayedPathNode
@@ -856,3 +856,34 @@ def task_example(
assert session.exit_code == ExitCode.OK
assert len(session.tasks) == 1
assert len(session.tasks[0].produces["return"]) == 2
+
+
+@pytest.mark.end_to_end()
+def test_task_that_depends_on_delayed_path_node(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode
+ from pathlib import Path
+ from pytask import task
+
+ def task_produces(
+ path = Path(__file__).parent
+ ) -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
+ path.joinpath("a.txt").write_text("Hello, ")
+ path.joinpath("b.txt").write_text("World!")
+
+ @task(is_ready=lambda x: Path(__file__).parent.joinpath("a.txt").exists())
+ def task_depends(
+ paths = DelayedPathNode(pattern="[ab].txt")
+ ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
+ path_dict = {path.stem: path for path in paths}
+ return path_dict["a"].read_text() + path_dict["b"].read_text()
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ session = build(paths=tmp_path)
+
+ assert session.exit_code == ExitCode.OK
+ assert len(session.tasks) == 2
+ assert len(session.tasks[0].produces["return"]) == 2
+ assert len(session.tasks[1].depends_on["paths"]) == 2
From 18318a3693cf5f69a077e7bd3fd113b7c71f2924 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 23 Oct 2023 11:06:48 +0200
Subject: [PATCH 08/81] Fix.
---
src/_pytask/collect.py | 12 ++++++++++++
src/_pytask/models.py | 6 ++++++
src/_pytask/session.py | 4 ++++
tests/test_collect.py | 18 ++++++++++++++++++
4 files changed, 40 insertions(+)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 0d905531..8714cdbd 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -26,6 +26,7 @@
from _pytask.exceptions import CollectionError
from _pytask.mark import MarkGenerator
from _pytask.mark_utils import has_mark
+from _pytask.models import DelayedTask
from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
@@ -249,6 +250,17 @@ def pytask_collect_task(
"""
if (name.startswith("task_") or has_mark(obj, "task")) and callable(obj):
+ if hasattr(obj, "pytask_meta") and obj.pytask_meta.is_ready is not None:
+ try:
+ is_ready = obj.pytask_meta.is_ready()
+ except Exception as e: # noqa: BLE001
+ msg = "The function for the 'is_ready' condition failed."
+ raise ValueError(msg) from e
+
+ if not is_ready:
+ session.delayed_tasks.append(DelayedTask(path=path, name=name, obj=obj))
+ return None
+
path_nodes = Path.cwd() if path is None else path.parent
dependencies = parse_dependencies_from_task_function(
session, path, name, path_nodes, obj
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index 2c2cebea..8e4de335 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -40,3 +40,9 @@ class NodeInfo(NamedTuple):
task_path: Path | None
task_name: str
value: Any
+
+
+class DelayedTask(NamedTuple):
+ path: Path | None
+ name: str
+ obj: Any
diff --git a/src/_pytask/session.py b/src/_pytask/session.py
index bde0f56a..e4b2d45a 100644
--- a/src/_pytask/session.py
+++ b/src/_pytask/session.py
@@ -12,6 +12,7 @@
if TYPE_CHECKING:
+ from _pytask.models import DelayedTask
from _pytask.node_protocols import PTask
from _pytask.warnings_utils import WarningReport
from _pytask.report import CollectionReport
@@ -31,6 +32,8 @@ class Session:
Reports for collected items.
dag
The DAG of the project.
+ delayed_tasks
+ List of all delayed tasks that are collected once they are ready.
hook
Holds all hooks collected by pytask.
tasks
@@ -51,6 +54,7 @@ class Session:
config: dict[str, Any] = field(factory=dict)
collection_reports: list[CollectionReport] = field(factory=list)
dag: nx.DiGraph = field(factory=nx.DiGraph)
+ delayed_tasks: list[DelayedTask] = field(factory=list)
hook: HookRelay = field(factory=HookRelay)
tasks: list[PTask] = field(factory=list)
dag_reports: DagReport | None = None
diff --git a/tests/test_collect.py b/tests/test_collect.py
index 7f9c1c28..cf94350e 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -231,6 +231,7 @@ def test_pytask_collect_node_does_not_raise_error_if_path_is_not_normalized(
tmp_path,
NodeInfo(
arg_name="",
+ allow_delayed=False,
path=(),
value=collected_node,
task_path=tmp_path / "task_example.py",
@@ -563,3 +564,20 @@ def task_example(a = DelayedPathNode(pattern="*.txt")): ...
result = runner.invoke(cli, [tmp_path.as_posix()])
assert result.exit_code == ExitCode.COLLECTION_FAILED
assert "Only a delayed task can depend on a delayed dependency." in result.output
+
+
+@pytest.mark.end_to_end()
+def test_gracefully_fail_with_failing_is_ready_condition(runner, tmp_path):
+ source = """
+ from pytask import task
+
+ def raise_(): raise Exception("ERROR")
+
+ @task(is_ready=raise_)
+ def task_example(): ...
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.COLLECTION_FAILED
+ assert "The function for the 'is_ready' condition failed." in result.output
From b8a16c4851bdd80bbf3a4a12f6524890c237238c Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 23 Oct 2023 13:19:19 +0200
Subject: [PATCH 09/81] fix.
---
src/pytask/__init__.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py
index bf6d4cfe..50db1a7b 100644
--- a/src/pytask/__init__.py
+++ b/src/pytask/__init__.py
@@ -108,9 +108,6 @@
"PTask",
"PTaskWithPath",
"PathNode",
- "PathNode",
- "PathNode",
- "Persisted",
"Persisted",
"Product",
"PytaskError",
From c96d58d99291a6b1c5f592ad7deae4a267f13f6a Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 25 Oct 2023 18:57:40 +0200
Subject: [PATCH 10/81] Add draft.
---
src/_pytask/delayed.py | 54 ++++++++++++++++++++++++++++++++++++++++
src/_pytask/execute.py | 2 ++
src/_pytask/hookspecs.py | 5 ++++
tests/test_collect.py | 2 +-
tests/test_execute.py | 2 +-
5 files changed, 63 insertions(+), 2 deletions(-)
create mode 100644 src/_pytask/delayed.py
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
new file mode 100644
index 00000000..0c65cec5
--- /dev/null
+++ b/src/_pytask/delayed.py
@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from _pytask.config import hookimpl
+from _pytask.node_protocols import PTask
+from _pytask.outcomes import CollectionOutcome
+
+if TYPE_CHECKING:
+ from _pytask.session import Session
+
+
+@hookimpl
+def pytask_execute_collect_delayed_tasks(session: Session) -> None:
+ """Collect tasks that are delayed."""
+ # Separate ready and delayed tasks.
+ still_delayed_tasks = []
+ ready_tasks = []
+
+ for delayed_task in session.delayed_tasks:
+ if delayed_task.obj.pytask_meta.is_ready():
+ ready_tasks.append(delayed_task)
+ else:
+ still_delayed_tasks.append(delayed_task)
+
+ session.delayed_tasks = still_delayed_tasks
+
+ if not ready_tasks:
+ return
+
+ # Collect tasks that are now ready.
+ new_collection_reports = []
+ for ready_task in ready_tasks:
+ report = session.hook.pytask_collect_task_protocol(
+ session=session,
+ reports=session.collection_reports,
+ path=ready_task.path,
+ name=ready_task.name,
+ obj=ready_task.obj,
+ )
+
+ if report is not None:
+ new_collection_reports.append(report)
+
+ # What to do with failed tasks? Can we just add them to executionreports.
+
+ # Add new tasks.
+ session.tasks.extend(
+ i.node
+ for i in session.collection_reports
+ if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, PTask)
+ )
+
+ session.hook.pytask_dag(session=session)
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 7946126f..3528a834 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -91,6 +91,8 @@ def pytask_execute_build(session: Session) -> bool | None:
session.execution_reports.append(report)
session.scheduler.done(task_name)
+ session.hook.pytask_execute_collect_delayed_tasks(session=session)
+
if session.should_stop:
return True
return True
diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py
index b1b3c522..260c18ae 100644
--- a/src/_pytask/hookspecs.py
+++ b/src/_pytask/hookspecs.py
@@ -390,6 +390,11 @@ def pytask_execute_task_log_end(session: Session, report: ExecutionReport) -> No
"""Log the end of a task execution."""
+@hookspec
+def pytask_execute_collect_delayed_tasks(session: Session) -> list[PTask]:
+ """Collect delayed tasks."""
+
+
@hookspec
def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> None:
"""Log the footer of the execution report."""
diff --git a/tests/test_collect.py b/tests/test_collect.py
index cf94350e..333d009d 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -563,7 +563,7 @@ def task_example(a = DelayedPathNode(pattern="*.txt")): ...
result = runner.invoke(cli, [tmp_path.as_posix()])
assert result.exit_code == ExitCode.COLLECTION_FAILED
- assert "Only a delayed task can depend on a delayed dependency." in result.output
+ assert "Only a delayed task can depend on a delayed node." in result.output
@pytest.mark.end_to_end()
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 867004e5..cb8224f0 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -874,7 +874,7 @@ def task_produces(
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
- @task(is_ready=lambda x: Path(__file__).parent.joinpath("a.txt").exists())
+ @task(is_ready=lambda *x: Path(__file__).parent.joinpath("a.txt").exists())
def task_depends(
paths = DelayedPathNode(pattern="[ab].txt")
) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
From ecb2e6044ebed108069f57e150a8b5d61b635491 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 8 Nov 2023 10:43:43 +0100
Subject: [PATCH 11/81] fix.
---
tests/test_dag_utils.py | 25 ++++++++++++++++++-------
1 file changed, 18 insertions(+), 7 deletions(-)
diff --git a/tests/test_dag_utils.py b/tests/test_dag_utils.py
index f8e290e3..f40b8b80 100644
--- a/tests/test_dag_utils.py
+++ b/tests/test_dag_utils.py
@@ -30,7 +30,12 @@ def dag():
@pytest.mark.unit()
def test_sort_tasks_topologically(dag):
- topo_ordering = list(TopologicalSorter.from_dag(dag).static_order())
+ sorter = TopologicalSorter.from_dag(dag)
+ topo_ordering = []
+ while sorter.is_active():
+ task_signature = sorter.get_ready()[0]
+ topo_ordering.append(task_signature)
+ sorter.done(task_signature)
topo_names = [dag.nodes[sig]["task"].name for sig in topo_ordering]
assert topo_names == [f".::{i}" for i in range(5)]
@@ -180,17 +185,23 @@ def test_ask_for_invalid_number_of_ready_tasks(dag):
@pytest.mark.unit()
def test_instantiate_sorter_from_other_sorter(dag):
+ name_to_signature = {
+ dag.nodes[signature]["task"].name: signature for signature in dag.nodes
+ }
+
scheduler = TopologicalSorter.from_dag(dag)
for _ in range(2):
- task_name = scheduler.get_ready()[0]
- scheduler.done(task_name)
- assert scheduler._nodes_done == {".::0", ".::1"}
+ task_signature = scheduler.get_ready()[0]
+ scheduler.done(task_signature)
+ assert scheduler._nodes_done == {
+ name_to_signature[name] for name in (".::0", ".::1")
+ }
dag.add_node(".::5", task=Task(base_name="5", path=Path(), function=None))
dag.add_edge(".::4", ".::5")
new_scheduler = TopologicalSorter.from_dag_and_sorter(dag, scheduler)
while new_scheduler.is_active():
- task_name = new_scheduler.get_ready()[0]
- new_scheduler.done(task_name)
- assert new_scheduler._nodes_done == {".::0", ".::1", ".::2", ".::3", ".::4", ".::5"}
+ task_signature = new_scheduler.get_ready()[0]
+ new_scheduler.done(task_signature)
+ assert new_scheduler._nodes_done == set(name_to_signature.values())
From 41c76cf02d7e5c2825b8ed10e603411b8db0f4ff Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 8 Nov 2023 18:26:25 +0100
Subject: [PATCH 12/81] Temp.
---
src/_pytask/nodes.py | 6 ++++++
src/_pytask/traceback.py | 6 +++---
tests/test_execute.py | 3 +--
tests/test_nodes.py | 1 +
4 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 46829564..0436a7e9 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -358,6 +358,12 @@ class DelayedPathNode(PNode):
pattern: str
name: str = ""
+ @property
+ def signature(self) -> str:
+ """The unique signature of the node."""
+ raw_key = "".join(str(hash_value(arg)) for arg in (self.root_dir, self.pattern))
+ return hashlib.sha256(raw_key.encode()).hexdigest()
+
def load(self, is_product: bool = False) -> None:
raise NotImplementedError
diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py
index c3f0ea61..d476f39d 100644
--- a/src/_pytask/traceback.py
+++ b/src/_pytask/traceback.py
@@ -47,9 +47,9 @@ class Traceback:
show_locals: ClassVar[bool] = False
suppress: ClassVar[tuple[Path, ...]] = (
- _PLUGGY_DIRECTORY,
- TREE_UTIL_LIB_DIRECTORY,
- _PYTASK_DIRECTORY,
+ # _PLUGGY_DIRECTORY,
+ # TREE_UTIL_LIB_DIRECTORY,
+ # _PYTASK_DIRECTORY,
)
def __rich_console__(
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 2763e627..3b2f5c24 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1071,9 +1071,8 @@ def test_task_that_produces_delayed_path_node(tmp_path):
from pathlib import Path
- def task_example(
+ def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
path = Path(__file__).parent
- ) -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
path.joinpath("a.txt").touch()
path.joinpath("b.txt").touch()
"""
diff --git a/tests/test_nodes.py b/tests/test_nodes.py
index b396e326..b225e551 100644
--- a/tests/test_nodes.py
+++ b/tests/test_nodes.py
@@ -52,6 +52,7 @@ def test_hash_of_python_node(value, hash_, expected):
value=None,
task_path=Path("task_example.py"),
task_name="task_example",
+ allow_delayed=False,
),
),
"7284475a87b8f1aa49c40126c5064269f0ba926265b8fe9158a39a882c6a1512",
From e71bc51487fd4c1bf2d94d740315445e1afd31c2 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Thu, 9 Nov 2023 13:45:11 +0100
Subject: [PATCH 13/81] Allow executing tasks with delayednodes.
---
src/_pytask/collect_utils.py | 5 ++--
src/_pytask/execute.py | 47 ++++++++++++++++++++++++++++++++++--
src/_pytask/traceback.py | 6 ++---
tests/test_execute.py | 3 +--
4 files changed, 52 insertions(+), 9 deletions(-)
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index 074cd86b..822d717e 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -42,6 +42,7 @@
__all__ = [
+ "collect_dependency",
"depends_on",
"parse_dependencies_from_task_function",
"parse_nodes",
@@ -317,7 +318,7 @@ def parse_dependencies_from_task_function(
continue
nodes = tree_map_with_path(
- lambda p, x: _collect_dependency(
+ lambda p, x: collect_dependency(
session,
node_path,
task_name,
@@ -616,7 +617,7 @@ def _collect_decorator_node(
return collected_node
-def _collect_dependency(
+def collect_dependency(
session: Session, path: Path, name: str, node_info: NodeInfo
) -> PNode:
"""Collect nodes for a task.
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 5f12f41d..b0d7573b 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -4,9 +4,11 @@
import inspect
import sys
import time
+from pathlib import Path
from typing import Any
from typing import TYPE_CHECKING
+from _pytask.collect_utils import collect_dependency
from _pytask.config import hookimpl
from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE
from _pytask.console import console
@@ -23,18 +25,23 @@
from _pytask.exceptions import NodeNotFoundError
from _pytask.mark import Mark
from _pytask.mark_utils import has_mark
+from _pytask.models import NodeInfo
from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
+from _pytask.node_protocols import PTaskWithPath
+from _pytask.nodes import Task
from _pytask.outcomes import count_outcomes
from _pytask.outcomes import Exit
from _pytask.outcomes import TaskOutcome
from _pytask.outcomes import WouldBeExecuted
from _pytask.reports import ExecutionReport
from _pytask.traceback import remove_traceback_from_exc_info
+from _pytask.tree_util import PyTree
from _pytask.tree_util import tree_leaves
from _pytask.tree_util import tree_map
+from _pytask.tree_util import tree_map_with_path
from _pytask.tree_util import tree_structure
from rich.text import Text
@@ -197,13 +204,49 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
return True
+def _collect_delayed_nodes(
+ session: Session, task: PTask, node: Any, path: tuple[Any, ...]
+) -> PyTree[PNode]:
+ """Collect delayed nodes."""
+ if not isinstance(node, PDelayedNode):
+ return node
+
+ node_path = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd()
+ task_name = task.base_name if isinstance(task, Task) else task.name
+ task_path = task.path if isinstance(task, PTaskWithPath) else None
+ arg_name, *rest_path = path
+
+ delayed_nodes = node.collect()
+ return tree_map_with_path( # type: ignore[return-value]
+ lambda p, x: collect_dependency(
+ session,
+ node_path,
+ task_name,
+ NodeInfo(
+ arg_name=arg_name,
+ allow_delayed=False,
+ path=(*rest_path, *p),
+ value=x,
+ task_path=task_path,
+ task_name=task_name,
+ ),
+ ),
+ delayed_nodes,
+ )
+
+
@hookimpl
def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
"""Check if nodes are produced by a task."""
# Replace delayed nodes with their actually resolved nodes.
- task.produces = tree_map( # type: ignore[assignment]
- lambda x: x.collect() if isinstance(x, PDelayedNode) else x, task.produces
+ has_delayed_nodes = any(
+ isinstance(node, PDelayedNode) for node in tree_leaves(task.produces)
)
+ if has_delayed_nodes:
+ # Collect delayed nodes
+ task.produces = tree_map_with_path( # type: ignore[assignment]
+ lambda p, x: _collect_delayed_nodes(session, task, x, p), task.produces
+ )
missing_nodes = [node for node in tree_leaves(task.produces) if not node.state()] # type: ignore[attr-defined]
if missing_nodes:
diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py
index d476f39d..c3f0ea61 100644
--- a/src/_pytask/traceback.py
+++ b/src/_pytask/traceback.py
@@ -47,9 +47,9 @@ class Traceback:
show_locals: ClassVar[bool] = False
suppress: ClassVar[tuple[Path, ...]] = (
- # _PLUGGY_DIRECTORY,
- # TREE_UTIL_LIB_DIRECTORY,
- # _PYTASK_DIRECTORY,
+ _PLUGGY_DIRECTORY,
+ TREE_UTIL_LIB_DIRECTORY,
+ _PYTASK_DIRECTORY,
)
def __rich_console__(
diff --git a/tests/test_execute.py b/tests/test_execute.py
index f01abad5..b7bc71d5 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1093,9 +1093,8 @@ def test_task_that_depends_on_delayed_path_node(tmp_path):
from pathlib import Path
from pytask import task
- def task_produces(
+ def task_produces() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
path = Path(__file__).parent
- ) -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
From 3ca95df7d8b0dcc1476ceb70fbd4f78c2ffb0012 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 10 Nov 2023 00:56:46 +0100
Subject: [PATCH 14/81] Allow tasks to depend on delayed nodes.
---
src/_pytask/collect_utils.py | 18 ++++++++++-
src/_pytask/execute.py | 32 ++++++++++++++++++-
src/_pytask/node_protocols.py | 2 +-
src/_pytask/nodes.py | 10 +++---
tests/test_collect.py | 2 +-
tests/test_execute.py | 59 ++++++++++++++++++++++++++++++++---
6 files changed, 109 insertions(+), 14 deletions(-)
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index 822d717e..487cfd94 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -18,6 +18,7 @@
from _pytask.mark_utils import has_mark
from _pytask.mark_utils import remove_marks
from _pytask.models import NodeInfo
+from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.nodes import PythonNode
from _pytask.shared import find_duplicates
@@ -248,7 +249,7 @@ def _merge_dictionaries(list_of_dicts: list[dict[Any, Any]]) -> dict[Any, Any]:
"""
-def parse_dependencies_from_task_function(
+def parse_dependencies_from_task_function( # noqa: C901
session: Session, task_path: Path | None, task_name: str, node_path: Path, obj: Any
) -> dict[str, Any]:
"""Parse dependencies from task function."""
@@ -317,6 +318,21 @@ def parse_dependencies_from_task_function(
if parameter_name == "depends_on":
continue
+ # Check for delayed nodes.
+ has_delayed_nodes = any(isinstance(i, PDelayedNode) for i in tree_leaves(value))
+ if has_delayed_nodes and not allow_delayed:
+ msg = (
+ f"The task {task_name!r} is not marked as a delayed task, but it "
+ "depends on a delayed node. Use '@task(is_ready=...) to create a "
+ "delayed task."
+ )
+ raise ValueError(msg)
+
+ # Collect delayed nodes.
+ value = tree_map( # noqa: PLW2901
+ lambda x: x.collect(node_path) if isinstance(x, PDelayedNode) else x, value
+ )
+
nodes = tree_map_with_path(
lambda p, x: collect_dependency(
session,
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index b0d7573b..9c70c279 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -32,6 +32,7 @@
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
from _pytask.nodes import Task
+from _pytask.outcomes import CollectionOutcome
from _pytask.outcomes import count_outcomes
from _pytask.outcomes import Exit
from _pytask.outcomes import TaskOutcome
@@ -216,7 +217,7 @@ def _collect_delayed_nodes(
task_path = task.path if isinstance(task, PTaskWithPath) else None
arg_name, *rest_path = path
- delayed_nodes = node.collect()
+ delayed_nodes = node.collect(node_path)
return tree_map_with_path( # type: ignore[return-value]
lambda p, x: collect_dependency(
session,
@@ -247,6 +248,35 @@ def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
task.produces = tree_map_with_path( # type: ignore[assignment]
lambda p, x: _collect_delayed_nodes(session, task, x, p), task.produces
)
+ # Collect delayed tasks.
+ new_collection_reports = []
+ still_delayed_tasks = []
+ for delayed_task in session.delayed_tasks:
+ if delayed_task.obj.pytask_meta.is_ready():
+ report = session.hook.pytask_collect_task_protocol(
+ session=session,
+ reports=session.collection_reports,
+ path=delayed_task.path,
+ name=delayed_task.name,
+ obj=delayed_task.obj,
+ )
+ new_collection_reports.append(report)
+ else:
+ still_delayed_tasks.append(delayed_task)
+ session.delayed_tasks = still_delayed_tasks
+ session.collection_reports.extend(new_collection_reports)
+ session.tasks.extend(
+ i.node
+ for i in new_collection_reports
+ if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, PTask)
+ )
+
+ # Recreate the DAG.
+ session.hook.pytask_dag(session=session)
+ # Update scheduler.
+ session.scheduler = TopologicalSorter.from_dag_and_sorter(
+ dag=session.dag, sorter=session.scheduler
+ )
missing_nodes = [node for node in tree_leaves(task.produces) if not node.state()] # type: ignore[attr-defined]
if missing_nodes:
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index bf0903e2..aa74856b 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -110,5 +110,5 @@ class PDelayedNode(Protocol):
"""
- def collect(self) -> list[Any]:
+ def collect(self, node_path: Path) -> list[Any]:
"""Collect the objects that are defined by the fuzzy node."""
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 61f49c77..b74193c1 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -4,7 +4,6 @@
import hashlib
import inspect
import pickle
-from pathlib import Path
from typing import Any
from typing import Callable
from typing import TYPE_CHECKING
@@ -22,6 +21,7 @@
if TYPE_CHECKING:
+ from pathlib import Path
from _pytask.models import NodeInfo
from _pytask.tree_util import PyTree
from _pytask.mark import Mark
@@ -373,9 +373,7 @@ def save(self, value: Any) -> None:
def state(self) -> None:
return None
- def collect(self) -> list[Path]:
+ def collect(self, node_path: Path) -> list[Path]:
"""Collect paths defined by the pattern."""
- if not isinstance(self.root_dir, Path):
- msg = f"'root_dir' should be a 'pathlib.Path', but it is {self.root_dir!r}"
- raise TypeError(msg)
- return list(self.root_dir.glob(self.pattern))
+ reference_path = node_path if self.root_dir is None else self.root_dir
+ return list(reference_path.glob(self.pattern))
diff --git a/tests/test_collect.py b/tests/test_collect.py
index 41809f93..83783e57 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -588,7 +588,7 @@ def task_example(a = DelayedPathNode(pattern="*.txt")): ...
result = runner.invoke(cli, [tmp_path.as_posix()])
assert result.exit_code == ExitCode.COLLECTION_FAILED
- assert "Only a delayed task can depend on a delayed node." in result.output
+ assert "The task 'task_example' is not marked" in result.output
@pytest.mark.end_to_end()
diff --git a/tests/test_execute.py b/tests/test_execute.py
index b7bc71d5..53c82520 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1070,7 +1070,6 @@ def test_task_that_produces_delayed_path_node(tmp_path):
from pytask import DelayedPathNode
from pathlib import Path
-
def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").touch()
@@ -1086,12 +1085,64 @@ def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
@pytest.mark.end_to_end()
-def test_task_that_depends_on_delayed_path_node(tmp_path):
+def test_task_that_depends_on_relative_delayed_path_node(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode
+ from pytask import DelayedPathNode, task
+ from pathlib import Path
+
+ @task(is_ready=lambda *x: True)
+ def task_delayed(
+ paths = DelayedPathNode(pattern="[ab].txt")
+ ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
+ path_dict = {path.stem: path for path in paths}
+ return path_dict["a"].read_text() + path_dict["b"].read_text()
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+ tmp_path.joinpath("a.txt").write_text("Hello, ")
+ tmp_path.joinpath("b.txt").write_text("World!")
+
+ session = build(paths=tmp_path)
+
+ assert session.exit_code == ExitCode.OK
+ assert len(session.tasks) == 1
+ assert len(session.tasks[0].depends_on["paths"]) == 2
+
+
+@pytest.mark.end_to_end()
+def test_task_that_depends_on_delayed_path_node_with_root_dir(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task
+ from pathlib import Path
+
+ root_dir = Path(__file__).parent / "subfolder"
+
+ @task(is_ready=lambda *x: True)
+ def task_delayed(
+ paths = DelayedPathNode(root_dir=root_dir, pattern="[ab].txt")
+ ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
+ path_dict = {path.stem: path for path in paths}
+ return path_dict["a"].read_text() + path_dict["b"].read_text()
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+ tmp_path.joinpath("subfolder").mkdir()
+ tmp_path.joinpath("subfolder", "a.txt").write_text("Hello, ")
+ tmp_path.joinpath("subfolder", "b.txt").write_text("World!")
+
+ session = build(paths=tmp_path)
+
+ assert session.exit_code == ExitCode.OK
+ assert len(session.tasks) == 1
+ assert len(session.tasks[0].depends_on["paths"]) == 2
+
+
+@pytest.mark.end_to_end()
+def test_task_that_depends_on_delayed_task(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task
from pathlib import Path
- from pytask import task
def task_produces() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
path = Path(__file__).parent
From ca77a19631e75b8ebbdaabb142bc429e4695e348 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 10 Nov 2023 01:02:06 +0100
Subject: [PATCH 15/81] to changes.
---
docs/source/changes.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/docs/source/changes.md b/docs/source/changes.md
index 3202775e..2d5dee6e 100644
--- a/docs/source/changes.md
+++ b/docs/source/changes.md
@@ -13,6 +13,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
- {pull}`485` adds missing steps to unconfigure pytask after the job is done which
caused flaky tests.
- {pull}`486` adds default names to {class}`~pytask.PPathNode`.
+- {pull}`487` implements delayed tasks and nodes.
## 0.4.2 - 2023-11-8
From 43377001b50f080f14aeb74d14849f3211bd98a1 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 10 Nov 2023 01:05:15 +0100
Subject: [PATCH 16/81] fix.
---
src/_pytask/nodes.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index b74193c1..b1264d55 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -4,6 +4,7 @@
import hashlib
import inspect
import pickle
+from pathlib import Path # noqa: TCH003
from typing import Any
from typing import Callable
from typing import TYPE_CHECKING
@@ -21,7 +22,6 @@
if TYPE_CHECKING:
- from pathlib import Path
from _pytask.models import NodeInfo
from _pytask.tree_util import PyTree
from _pytask.mark import Mark
From c2db3c5ea09b8005411594a3b5d88794b3053948 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 10 Nov 2023 10:52:52 +0100
Subject: [PATCH 17/81] Fix.
---
tests/test_collect.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/tests/test_collect.py b/tests/test_collect.py
index 83783e57..c1e5e753 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -213,6 +213,7 @@ def test_pytask_collect_node_raises_error_if_path_is_not_correctly_cased(tmp_pat
tmp_path,
NodeInfo(
arg_name="",
+ allow_delayed=False,
path=(),
value=collected_node,
task_path=tmp_path.joinpath("task_example.py"),
From 104aa5f0c58220fce750bb3e2611d6353b6e8cb8 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 10 Nov 2023 13:29:27 +0100
Subject: [PATCH 18/81] remove allow_delayed.
---
src/_pytask/collect.py | 19 +++++--------------
src/_pytask/collect_utils.py | 8 --------
src/_pytask/data_catalog.py | 1 -
src/_pytask/execute.py | 1 -
src/_pytask/models.py | 1 -
tests/test_collect.py | 4 ----
tests/test_collect_command.py | 2 --
tests/test_nodes.py | 1 -
8 files changed, 5 insertions(+), 32 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index ec8937a2..e7f83f3d 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -25,7 +25,6 @@
from _pytask.mark import MarkGenerator
from _pytask.mark_utils import has_mark
from _pytask.models import DelayedTask
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
@@ -323,7 +322,7 @@ def pytask_collect_task(
@hookimpl(trylast=True)
-def pytask_collect_node( # noqa: C901, PLR0912
+def pytask_collect_node( # noqa: C901
session: Session, path: Path, node_info: NodeInfo
) -> PNode:
"""Collect a node of a task as a :class:`pytask.PNode`.
@@ -345,18 +344,10 @@ def pytask_collect_node( # noqa: C901, PLR0912
"""
node = node_info.value
- if isinstance(node, PDelayedNode):
- if not node_info.allow_delayed:
- msg = (
- "Only a delayed task can depend on a delayed node. The delayed "
- f"dependency is {node!r}."
- )
- raise ValueError(msg)
-
- if isinstance(node, DelayedPathNode):
- if node.root_dir is None:
- node.root_dir = path
- node.name = node.root_dir.joinpath(node.pattern).as_posix()
+ if isinstance(node, DelayedPathNode):
+ if node.root_dir is None:
+ node.root_dir = path
+ node.name = node.root_dir.joinpath(node.pattern).as_posix()
if isinstance(node, PythonNode):
node.node_info = node_info
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index 487cfd94..a2cd169f 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -99,7 +99,6 @@ def parse_nodes( # noqa: PLR0913
task_name,
NodeInfo(
arg_name=arg_name,
- allow_delayed=False,
path=(),
value=x,
task_path=task_path,
@@ -280,7 +279,6 @@ def parse_dependencies_from_task_function( # noqa: C901
task_name,
NodeInfo(
arg_name="depends_on",
- allow_delayed=allow_delayed,
path=(),
value=x,
task_path=task_path,
@@ -340,7 +338,6 @@ def parse_dependencies_from_task_function( # noqa: C901
task_name,
NodeInfo(
arg_name=parameter_name, # noqa: B023
- allow_delayed=allow_delayed,
path=p,
value=x,
task_path=task_path,
@@ -359,7 +356,6 @@ def parse_dependencies_from_task_function( # noqa: C901
node_name = create_name_of_python_node(
NodeInfo(
arg_name=parameter_name,
- allow_delayed=allow_delayed,
path=(),
value=value,
task_path=task_path,
@@ -449,7 +445,6 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="produces",
- allow_delayed=True,
path=p,
value=x,
task_path=task_path,
@@ -493,7 +488,6 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name=parameter_name, # noqa: B023
- allow_delayed=False,
path=p,
value=x,
task_path=task_path,
@@ -514,7 +508,6 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="return",
- allow_delayed=True,
path=p,
value=x,
task_path=task_path,
@@ -536,7 +529,6 @@ def parse_products_from_task_function(
task_name,
NodeInfo(
arg_name="return",
- allow_delayed=True,
path=p,
value=x,
task_path=task_path,
diff --git a/src/_pytask/data_catalog.py b/src/_pytask/data_catalog.py
index 5a31ffbb..ce8eb064 100644
--- a/src/_pytask/data_catalog.py
+++ b/src/_pytask/data_catalog.py
@@ -124,7 +124,6 @@ def add(self, name: str, node: DataCatalog | PNode | None = None) -> None:
value=node,
task_path=None,
task_name="",
- allow_delayed=False,
),
)
if collected_node is None: # pragma: no cover
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 9c70c279..e9f476fe 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -225,7 +225,6 @@ def _collect_delayed_nodes(
task_name,
NodeInfo(
arg_name=arg_name,
- allow_delayed=False,
path=(*rest_path, *p),
value=x,
task_path=task_path,
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index 8e4de335..b85f2af1 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -35,7 +35,6 @@ class CollectionMetadata:
class NodeInfo(NamedTuple):
arg_name: str
- allow_delayed: bool
path: tuple[str | int, ...]
task_path: Path | None
task_name: str
diff --git a/tests/test_collect.py b/tests/test_collect.py
index c1e5e753..a1963cab 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -162,7 +162,6 @@ def test_collect_files_w_custom_file_name_pattern(
Path(),
NodeInfo(
arg_name="",
- allow_delayed=False,
path=(),
value=Path.cwd() / "text.txt",
task_path=Path.cwd() / "task_example.py",
@@ -178,7 +177,6 @@ def test_collect_files_w_custom_file_name_pattern(
Path(),
NodeInfo(
arg_name="",
- allow_delayed=False,
path=(),
value=1,
task_path=Path.cwd() / "task_example.py",
@@ -213,7 +211,6 @@ def test_pytask_collect_node_raises_error_if_path_is_not_correctly_cased(tmp_pat
tmp_path,
NodeInfo(
arg_name="",
- allow_delayed=False,
path=(),
value=collected_node,
task_path=tmp_path.joinpath("task_example.py"),
@@ -240,7 +237,6 @@ def test_pytask_collect_node_does_not_raise_error_if_path_is_not_normalized(
tmp_path,
NodeInfo(
arg_name="",
- allow_delayed=False,
path=(),
value=collected_node,
task_path=tmp_path / "task_example.py",
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 759a4c17..5ebea170 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -697,9 +697,7 @@ def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]: ...
def test_collect_custom_node_receives_default_name(runner, tmp_path):
source = """
from typing_extensions import Annotated
- from attrs import define
- @define
class CustomNode:
name: str = ""
diff --git a/tests/test_nodes.py b/tests/test_nodes.py
index b225e551..b396e326 100644
--- a/tests/test_nodes.py
+++ b/tests/test_nodes.py
@@ -52,7 +52,6 @@ def test_hash_of_python_node(value, hash_, expected):
value=None,
task_path=Path("task_example.py"),
task_name="task_example",
- allow_delayed=False,
),
),
"7284475a87b8f1aa49c40126c5064269f0ba926265b8fe9158a39a882c6a1512",
From 9d70e3fd7f332002f05cfe767ec19dd22018d2c4 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 10 Nov 2023 16:43:51 +0100
Subject: [PATCH 19/81] Fix test.
---
tests/test_collect_command.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 5ebea170..1bdb2c73 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -701,8 +701,10 @@ def test_collect_custom_node_receives_default_name(runner, tmp_path):
class CustomNode:
name: str = ""
- def state(): return None
-
+ def state(self): return None
+ def signature(self): return "signature"
+ def load(self, is_product): ...
+ def save(self, value): ...
def task_example() -> Annotated[None, CustomNode()]: ...
"""
From 0928a7ec241c1f5c6c330bd47508f769f920bde2 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 10 Nov 2023 17:18:24 +0100
Subject: [PATCH 20/81] extend test.
---
tests/test_collect_command.py | 31 +++++++++++++++++++++++++------
1 file changed, 25 insertions(+), 6 deletions(-)
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 1bdb2c73..90ed7079 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -663,17 +663,23 @@ def task_example() -> Annotated[Dict[str, str], nodes]:
@pytest.mark.end_to_end()
-def test_collect_task_with_delayed_path_node(runner, tmp_path):
- source = """
- from pytask import DelayedPathNode
+@pytest.mark.parametrize('node_def', [
+ "paths: Annotated[list[Path], DelayedPathNode(pattern='*.txt'), Product])",
+ "produces=DelayedPathNode(pattern='*.txt'))",
+ ") -> Annotated[None, DelayedPathNode(pattern='*.txt')]",
+])
+def test_collect_task_with_delayed_path_node_as_product(runner, tmp_path, node_def):
+ source = f"""
+ from pytask import DelayedPathNode, Product
from typing_extensions import Annotated
+ from pathlib import Path
- def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]: ...
+ def task_example({node_def}: ...
"""
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+ # Without nodes.
result = runner.invoke(cli, ["collect", tmp_path.as_posix()])
-
assert result.exit_code == ExitCode.OK
captured = result.output.replace("\n", "").replace(" ", "")
assert "<Module" in captured
@@ -681,8 +687,8 @@ def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]: ...
assert "<Function" in captured
assert "task_example>" in captured
+ # With nodes.
result = runner.invoke(cli, ["collect", tmp_path.as_posix(), "--nodes"])
-
assert result.exit_code == ExitCode.OK
captured = result.output.replace("\n", "").replace(" ", "")
assert "<Module" in captured
@@ -692,6 +698,19 @@ def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]: ...
assert "<Product" in captured
assert "/*.txt>" in captured
+ # With existing nodes.
+ tmp_path.joinpath("a.txt").touch()
+ result = runner.invoke(cli, ["collect", tmp_path.as_posix(), "--nodes"])
+ assert result.exit_code == ExitCode.OK
+ captured = result.output.replace("\n", "").replace(" ", "")
+ assert "<Module" in captured
+ assert "task_module.py>" in captured
+ assert "<Function" in captured
+ assert "task_example>" in captured
+ assert "<Product" in captured
+ assert "/*.txt>" not in captured
+ assert "a.txt>" not in captured
+
@pytest.mark.end_to_end()
def test_collect_custom_node_receives_default_name(runner, tmp_path):
From f2d236bfec6e9a74d7cd6073a9374dbd5d0b8615 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 11 Nov 2023 20:23:18 +0100
Subject: [PATCH 21/81] Add new hook.
---
src/_pytask/collect.py | 2 ++
src/_pytask/hookspecs.py | 8 ++++++++
tests/test_collect_command.py | 13 ++++++++-----
3 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index a668b2dd..81e339ea 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -261,6 +261,8 @@ def pytask_collect_task(
return None
path_nodes = Path.cwd() if path is None else path.parent
+
+ # Collect dependencies and products.
dependencies = parse_dependencies_from_task_function(
session, path, name, path_nodes, obj
)
diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py
index 433908a0..303bb0b6 100644
--- a/src/_pytask/hookspecs.py
+++ b/src/_pytask/hookspecs.py
@@ -13,6 +13,7 @@
if TYPE_CHECKING:
+ from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import MetaNode
from _pytask.models import NodeInfo
from _pytask.node_protocols import PNode
@@ -194,6 +195,13 @@ def pytask_collect_task_teardown(session: Session, task: PTask) -> None:
"""
+@hookspec
+def pytask_collect_delayed_node(
+ session: Session, path: Path, node_info: NodeInfo
+) -> PDelayedNode | None:
+ """Collect a delayed node."""
+
+
@hookspec(firstresult=True)
def pytask_collect_node(
session: Session, path: Path, node_info: NodeInfo
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 90ed7079..138993f6 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -663,11 +663,14 @@ def task_example() -> Annotated[Dict[str, str], nodes]:
@pytest.mark.end_to_end()
-@pytest.mark.parametrize('node_def', [
- "paths: Annotated[list[Path], DelayedPathNode(pattern='*.txt'), Product])",
- "produces=DelayedPathNode(pattern='*.txt'))",
- ") -> Annotated[None, DelayedPathNode(pattern='*.txt')]",
-])
+@pytest.mark.parametrize(
+ "node_def",
+ [
+ "paths: Annotated[list[Path], DelayedPathNode(pattern='*.txt'), Product])",
+ "produces=DelayedPathNode(pattern='*.txt'))",
+ ") -> Annotated[None, DelayedPathNode(pattern='*.txt')]",
+ ],
+)
def test_collect_task_with_delayed_path_node_as_product(runner, tmp_path, node_def):
source = f"""
from pytask import DelayedPathNode, Product
From 3b8ef38ee7bb43472658f892b86f1e2e47ab57ae Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 11 Nov 2023 23:08:34 +0100
Subject: [PATCH 22/81] fix.
---
docs/source/changes.md | 2 +-
src/_pytask/collect.py | 60 ++++++++++++++++++++----------------
src/_pytask/collect_utils.py | 18 ++++++++++-
3 files changed, 51 insertions(+), 29 deletions(-)
diff --git a/docs/source/changes.md b/docs/source/changes.md
index a5bbdf09..c847745a 100644
--- a/docs/source/changes.md
+++ b/docs/source/changes.md
@@ -20,7 +20,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
any node.
- {pull}`490` refactors and better tests parsing of dependencies.
-## 0.4.2 - 2023-11-8
+## 0.4.2 - 2023-11-08
- {pull}`449` simplifies the code building the plugin manager.
- {pull}`451` improves `collect_command.py` and renames `graph.py` to `dag_command.py`.
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 81e339ea..3fd4b4da 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -21,11 +21,11 @@
from _pytask.console import create_summary_panel
from _pytask.console import get_file
from _pytask.console import is_jupyter
-from _pytask.exceptions import CollectionError
+from _pytask.exceptions import CollectionError, NodeNotCollectedError
from _pytask.mark import MarkGenerator
from _pytask.mark_utils import has_mark
from _pytask.models import DelayedTask
-from _pytask.node_protocols import PNode
+from _pytask.node_protocols import PDelayedNode, PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
from _pytask.nodes import DelayedPathNode
@@ -297,26 +297,15 @@ def pytask_collect_task(
return None
-_TEMPLATE_ERROR: str = """\
-The provided path of the dependency/product is
-
-{}
-
-, but the path of the file on disk is
-
-{}
-
-Case-sensitive file systems would raise an error because the upper and lower case \
-format of the paths does not match.
-
-Please, align the names to ensure reproducibility on case-sensitive file systems \
-(often Linux or macOS) or disable this error with 'check_casing_of_paths = false' in \
-your pytask configuration file.
-
-Hint: If parts of the path preceding your project directory are not properly \
-formatted, check whether you need to call `.resolve()` on `SRC`, `BLD` or other paths \
-created from the `__file__` attribute of a module.
-"""
+@hookimpl(trylast=True)
+def pytask_collect_delayed_node(session: Session, path: Path, node_info: NodeInfo) -> PDelayedNode:
+ """Collect a delayed node."""
+ node = node_info.value
+ if isinstance(node, DelayedPathNode):
+ if node.root_dir is None:
+ node.root_dir = path
+ node.name = node.root_dir.joinpath(node.pattern).as_posix()
+ return node
_TEMPLATE_ERROR_DIRECTORY: str = """\
@@ -346,11 +335,6 @@ def pytask_collect_node( # noqa: C901, PLR0912
"""
node = node_info.value
- if isinstance(node, DelayedPathNode):
- if node.root_dir is None:
- node.root_dir = path
- node.name = node.root_dir.joinpath(node.pattern).as_posix()
-
if isinstance(node, PythonNode):
node.node_info = node_info
if not node.name:
@@ -416,6 +400,28 @@ def pytask_collect_node( # noqa: C901, PLR0912
return PythonNode(value=node, name=node_name, node_info=node_info)
+_TEMPLATE_ERROR: str = """\
+The provided path of the dependency/product is
+
+{}
+
+, but the path of the file on disk is
+
+{}
+
+Case-sensitive file systems would raise an error because the upper and lower case \
+format of the paths does not match.
+
+Please, align the names to ensure reproducibility on case-sensitive file systems \
+(often Linux or macOS) or disable this error with 'check_casing_of_paths = false' in \
+your pytask configuration file.
+
+Hint: If parts of the path preceding your project directory are not properly \
+formatted, check whether you need to call `.resolve()` on `SRC`, `BLD` or other paths \
+created from the `__file__` attribute of a module.
+"""
+
+
def _raise_error_if_casing_of_path_is_wrong(
path: Path, check_casing_of_paths: bool
) -> None:
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index aad8faf0..4c7f4e04 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -451,6 +451,22 @@ def parse_products_from_task_function( # noqa: C901
parameter_name
)
+ preliminary_collected_products = tree_map_with_path(
+ lambda p, x: session.hook.pytask_collect_delayed_node(
+ session,
+ node_path,
+ task_name,
+ NodeInfo(
+ arg_name=parameter_name, # noqa: B023
+ path=p,
+ value=x,
+ task_path=task_path,
+ task_name=task_name,
+ ),
+ ) if isinstance(x, PDelayedNode) else x,
+ value,
+ )
+
collected_products = tree_map_with_path(
lambda p, x: _collect_product(
session,
@@ -464,7 +480,7 @@ def parse_products_from_task_function( # noqa: C901
task_name=task_name,
),
),
- value,
+ preliminary_collected_products,
)
out[parameter_name] = collected_products
From 948b743e94fac21768b5e06ef479cf438f70f91b Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 12 Nov 2023 19:10:59 +0100
Subject: [PATCH 23/81] Fixes.
---
src/_pytask/collect.py | 9 +--
src/_pytask/collect_utils.py | 130 +++++++++++++++++-----------------
src/_pytask/execute.py | 1 +
src/_pytask/hookspecs.py | 2 +-
src/_pytask/nodes.py | 7 +-
tests/test_collect_command.py | 13 ----
tests/test_execute.py | 11 +--
7 files changed, 83 insertions(+), 90 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 3fd4b4da..cf851cd7 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -21,11 +21,12 @@
from _pytask.console import create_summary_panel
from _pytask.console import get_file
from _pytask.console import is_jupyter
-from _pytask.exceptions import CollectionError, NodeNotCollectedError
+from _pytask.exceptions import CollectionError
from _pytask.mark import MarkGenerator
from _pytask.mark_utils import has_mark
from _pytask.models import DelayedTask
-from _pytask.node_protocols import PDelayedNode, PNode
+from _pytask.node_protocols import PDelayedNode
+from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
from _pytask.nodes import DelayedPathNode
@@ -298,7 +299,7 @@ def pytask_collect_task(
@hookimpl(trylast=True)
-def pytask_collect_delayed_node(session: Session, path: Path, node_info: NodeInfo) -> PDelayedNode:
+def pytask_collect_delayed_node(path: Path, node_info: NodeInfo) -> PDelayedNode:
"""Collect a delayed node."""
node = node_info.value
if isinstance(node, DelayedPathNode):
@@ -313,7 +314,7 @@ def pytask_collect_delayed_node(session: Session, path: Path, node_info: NodeInf
@hookimpl(trylast=True)
-def pytask_collect_node( # noqa: C901, PLR0912
+def pytask_collect_node( # noqa: C901
session: Session, path: Path, node_info: NodeInfo
) -> PNode:
"""Collect a node of a task as a :class:`pytask.PNode`.
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index c4941b5e..f8172d95 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -277,6 +277,7 @@ def parse_dependencies_from_task_function(
raise NodeNotCollectedError(_ERROR_MULTIPLE_DEPENDENCY_DEFINITIONS)
parameters_with_product_annot = _find_args_with_product_annotation(obj)
+ parameters_with_product_annot.append("return")
parameters_with_node_annot = _find_args_with_node_annotation(obj)
# Complete kwargs with node annotations, when no value is given by kwargs.
@@ -292,10 +293,7 @@ def parse_dependencies_from_task_function(
raise ValueError(msg)
for parameter_name, value in kwargs.items():
- if (
- parameter_name in parameters_with_product_annot
- or parameter_name == "return"
- ):
+ if parameter_name in parameters_with_product_annot:
continue
# Check for delayed nodes.
@@ -308,24 +306,13 @@ def parse_dependencies_from_task_function(
)
raise ValueError(msg)
- # Collect delayed nodes.
- value = tree_map( # noqa: PLW2901
- lambda x: x.collect(node_path) if isinstance(x, PDelayedNode) else x, value
- )
-
- nodes = tree_map_with_path(
- lambda p, x: collect_dependency(
- session,
- node_path,
- task_name,
- NodeInfo(
- arg_name=parameter_name, # noqa: B023
- path=p,
- value=x,
- task_path=task_path,
- task_name=task_name,
- ),
- ),
+ nodes = _collect_delayed_nodes_and_nodes(
+ collect_dependency,
+ session,
+ node_path,
+ task_name,
+ task_path,
+ parameter_name,
value,
)
@@ -456,56 +443,27 @@ def parse_products_from_task_function( # noqa: C901
value = kwargs.get(parameter_name) or parameters_with_node_annot.get(
parameter_name
)
-
- preliminary_collected_products = tree_map_with_path(
- lambda p, x: session.hook.pytask_collect_delayed_node(
- session,
- node_path,
- task_name,
- NodeInfo(
- arg_name=parameter_name, # noqa: B023
- path=p,
- value=x,
- task_path=task_path,
- task_name=task_name,
- ),
- ) if isinstance(x, PDelayedNode) else x,
+ collected_products = _collect_delayed_nodes_and_nodes(
+ _collect_product,
+ session,
+ node_path,
+ task_name,
+ task_path,
+ parameter_name,
value,
)
-
- collected_products = tree_map_with_path(
- lambda p, x: _collect_product(
- session,
- node_path,
- task_name,
- NodeInfo(
- arg_name=parameter_name, # noqa: B023
- path=p,
- value=x,
- task_path=task_path,
- task_name=task_name,
- ),
- ),
- preliminary_collected_products,
- )
out[parameter_name] = collected_products
task_produces = obj.pytask_meta.produces if hasattr(obj, "pytask_meta") else None
if task_produces:
has_task_decorator = True
- collected_products = tree_map_with_path(
- lambda p, x: _collect_product(
- session,
- node_path,
- task_name,
- NodeInfo(
- arg_name="return",
- path=p,
- value=x,
- task_path=task_path,
- task_name=task_name,
- ),
- ),
+ collected_products = _collect_delayed_nodes_and_nodes(
+ _collect_product,
+ session,
+ node_path,
+ task_name,
+ task_path,
+ "return",
task_produces,
)
out = {"return": collected_products}
@@ -527,6 +485,48 @@ def parse_products_from_task_function( # noqa: C901
return out
+def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
+ collection_func: Callable[..., Any],
+ session: Session,
+ node_path: Path,
+ task_name: str,
+ task_path: Path | None,
+ parameter_name: str,
+ value: Any,
+) -> PyTree[PDelayedNode | PNode]:
+ nodes = tree_map_with_path(
+ lambda p, x: session.hook.pytask_collect_delayed_node(
+ session=session,
+ path=node_path,
+ node_info=NodeInfo(
+ arg_name=parameter_name,
+ path=p,
+ value=x,
+ task_path=task_path,
+ task_name=task_name,
+ ),
+ )
+ if isinstance(x, PDelayedNode)
+ else x,
+ value,
+ )
+ return tree_map_with_path( # type: ignore[return-value]
+ lambda p, x: collection_func(
+ session,
+ node_path,
+ task_name,
+ NodeInfo(
+ arg_name=parameter_name,
+ path=p,
+ value=x,
+ task_path=task_path,
+ task_name=task_name,
+ ),
+ ),
+ nodes,
+ )
+
+
def _find_args_with_product_annotation(func: Callable[..., Any]) -> list[str]:
"""Find args with product annotations."""
annotations = get_annotations(func, eval_str=True)
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index e9f476fe..e2fde5a1 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -187,6 +187,7 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
if "return" in task.produces:
structure_out = tree_structure(out)
structure_return = tree_structure(task.produces["return"])
+
# strict must be false when none is leaf.
if not structure_return.is_prefix(structure_out, strict=False):
msg = (
diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py
index 303bb0b6..f506d150 100644
--- a/src/_pytask/hookspecs.py
+++ b/src/_pytask/hookspecs.py
@@ -195,7 +195,7 @@ def pytask_collect_task_teardown(session: Session, task: PTask) -> None:
"""
-@hookspec
+@hookspec(firstresult=True)
def pytask_collect_delayed_node(
session: Session, path: Path, node_info: NodeInfo
) -> PDelayedNode | None:
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index b1264d55..e1d21ac9 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -364,8 +364,11 @@ def signature(self) -> str:
raw_key = "".join(str(hash_value(arg)) for arg in (self.root_dir, self.pattern))
return hashlib.sha256(raw_key.encode()).hexdigest()
- def load(self, is_product: bool = False) -> None:
- raise NotImplementedError
+ def load(self, is_product: bool = False) -> Path | tuple[Path, str]:
+ assert self.root_dir
+ if is_product:
+ return self.root_dir
+ return self.root_dir, self.pattern
def save(self, value: Any) -> None:
raise NotImplementedError
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 138993f6..361375b3 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -701,19 +701,6 @@ def task_example({node_def}: ...
assert "<Product" in captured
assert "/*.txt>" in captured
- # With existing nodes.
- tmp_path.joinpath("a.txt").touch()
- result = runner.invoke(cli, ["collect", tmp_path.as_posix(), "--nodes"])
- assert result.exit_code == ExitCode.OK
- captured = result.output.replace("\n", "").replace(" ", "")
- assert "<Module" in captured
- assert "task_module.py>" in captured
- assert "<Function" in captured
- assert "task_example>" in captured
- assert "<Product" in captured
- assert "/*.txt>" not in captured
- assert "a.txt>" not in captured
-
@pytest.mark.end_to_end()
def test_collect_custom_node_receives_default_name(runner, tmp_path):
diff --git a/tests/test_execute.py b/tests/test_execute.py
index f9e804d9..4748124d 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -994,13 +994,14 @@ def task_example(a = PythonNode(value={"a": 1}, hash=True)):
def test_task_that_produces_delayed_path_node(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode
+ from pytask import DelayedPathNode, Product
from pathlib import Path
- def task_example() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
- path = Path(__file__).parent
- path.joinpath("a.txt").touch()
- path.joinpath("b.txt").touch()
+ def task_example(
+ root_path: Annotated[Path, DelayedPathNode(pattern="*.txt"), Product]
+ ):
+ root_path.joinpath("a.txt").write_text("Hello, ")
+ root_path.joinpath("b.txt").write_text("World!")
"""
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
From 9d14eff85495ee1545c24f2ce48fe3c00917b987 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 12 Nov 2023 21:36:17 +0100
Subject: [PATCH 24/81] fix.
---
src/_pytask/cli.py | 2 ++
src/_pytask/collect.py | 19 +++++--------------
src/_pytask/collect_utils.py | 9 ++++++++-
src/_pytask/delayed.py | 17 ++++++++++++++++-
src/_pytask/execute.py | 2 +-
src/_pytask/node_protocols.py | 2 +-
src/_pytask/nodes.py | 5 ++---
tests/test_collect_command.py | 28 ++++++++++++++++++++++++++++
tests/test_execute.py | 4 ++--
9 files changed, 65 insertions(+), 23 deletions(-)
diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py
index f056b2e1..236a6b94 100644
--- a/src/_pytask/cli.py
+++ b/src/_pytask/cli.py
@@ -53,6 +53,7 @@ def pytask_add_hooks(pm: pluggy.PluginManager) -> None:
from _pytask import config
from _pytask import database
from _pytask import debugging
+ from _pytask import delayed
from _pytask import execute
from _pytask import dag_command
from _pytask import live
@@ -75,6 +76,7 @@ def pytask_add_hooks(pm: pluggy.PluginManager) -> None:
pm.register(config)
pm.register(database)
pm.register(debugging)
+ pm.register(delayed)
pm.register(execute)
pm.register(dag_command)
pm.register(live)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index cf851cd7..a73ac4cf 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -25,11 +25,9 @@
from _pytask.mark import MarkGenerator
from _pytask.mark_utils import has_mark
from _pytask.models import DelayedTask
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
-from _pytask.nodes import DelayedPathNode
from _pytask.nodes import PathNode
from _pytask.nodes import PythonNode
from _pytask.nodes import Task
@@ -298,17 +296,6 @@ def pytask_collect_task(
return None
-@hookimpl(trylast=True)
-def pytask_collect_delayed_node(path: Path, node_info: NodeInfo) -> PDelayedNode:
- """Collect a delayed node."""
- node = node_info.value
- if isinstance(node, DelayedPathNode):
- if node.root_dir is None:
- node.root_dir = path
- node.name = node.root_dir.joinpath(node.pattern).as_posix()
- return node
-
-
_TEMPLATE_ERROR_DIRECTORY: str = """\
The path '{path}' points to a directory, although only files are allowed."""
@@ -505,7 +492,11 @@ def pytask_collect_log(
"""Log collection."""
session.collection_end = time.time()
- console.print(f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.")
+ msg = f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}."
+ if session.delayed_tasks:
+ n_delayed_tasks = len(session.delayed_tasks)
+ msg = msg[:-1] + f" and {n_delayed_tasks} delayed task{'' if n_delayed_tasks == 1 else 's'}."
+ console.print(msg)
failed_reports = [r for r in reports if r.outcome == CollectionOutcome.FAIL]
if failed_reports:
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index f8172d95..28f1f141 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -265,6 +265,7 @@ def parse_dependencies_from_task_function(
allow_delayed = (
obj.pytask_meta.is_ready is not None if hasattr(obj, "pytask_meta") else False
)
+ is_ready = obj.pytask_meta.is_ready() if allow_delayed else False
signature_defaults = parse_keyword_arguments_from_signature_defaults(obj)
kwargs = {**signature_defaults, **task_kwargs}
kwargs.pop("produces", None)
@@ -308,6 +309,7 @@ def parse_dependencies_from_task_function(
nodes = _collect_delayed_nodes_and_nodes(
collect_dependency,
+ is_ready,
session,
node_path,
task_name,
@@ -445,6 +447,7 @@ def parse_products_from_task_function( # noqa: C901
)
collected_products = _collect_delayed_nodes_and_nodes(
_collect_product,
+ False,
session,
node_path,
task_name,
@@ -459,6 +462,7 @@ def parse_products_from_task_function( # noqa: C901
has_task_decorator = True
collected_products = _collect_delayed_nodes_and_nodes(
_collect_product,
+ False,
session,
node_path,
task_name,
@@ -487,6 +491,7 @@ def parse_products_from_task_function( # noqa: C901
def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
collection_func: Callable[..., Any],
+ is_ready: bool,
session: Session,
node_path: Path,
task_name: str,
@@ -510,6 +515,8 @@ def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
else x,
value,
)
+ if is_ready:
+ nodes = tree_map(lambda x: x.collect() if isinstance(x, PDelayedNode) else x, nodes)
return tree_map_with_path( # type: ignore[return-value]
lambda p, x: collection_func(
session,
@@ -522,7 +529,7 @@ def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
task_path=task_path,
task_name=task_name,
),
- ),
+ ) if not isinstance(x, PDelayedNode) else x,
nodes,
)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 0c65cec5..94ae82e0 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -1,15 +1,30 @@
from __future__ import annotations
+from pathlib import Path
from typing import TYPE_CHECKING
from _pytask.config import hookimpl
-from _pytask.node_protocols import PTask
+from _pytask.models import NodeInfo
+from _pytask.node_protocols import PDelayedNode, PTask
+from _pytask.nodes import DelayedPathNode
from _pytask.outcomes import CollectionOutcome
if TYPE_CHECKING:
from _pytask.session import Session
+@hookimpl(trylast=True)
+def pytask_collect_delayed_node(path: Path, node_info: NodeInfo) -> PDelayedNode:
+ """Collect a delayed node."""
+ node = node_info.value
+ if isinstance(node, DelayedPathNode):
+ if node.root_dir is None:
+ node.root_dir = path
+ if not node.name:
+ node.name = node.root_dir.joinpath(node.pattern).as_posix()
+ return node
+
+
@hookimpl
def pytask_execute_collect_delayed_tasks(session: Session) -> None:
"""Collect tasks that are delayed."""
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index e2fde5a1..b330b5d7 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -218,7 +218,7 @@ def _collect_delayed_nodes(
task_path = task.path if isinstance(task, PTaskWithPath) else None
arg_name, *rest_path = path
- delayed_nodes = node.collect(node_path)
+ delayed_nodes = node.collect()
return tree_map_with_path( # type: ignore[return-value]
lambda p, x: collect_dependency(
session,
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index aa74856b..bf0903e2 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -110,5 +110,5 @@ class PDelayedNode(Protocol):
"""
- def collect(self, node_path: Path) -> list[Any]:
+ def collect(self) -> list[Any]:
"""Collect the objects that are defined by the fuzzy node."""
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index e1d21ac9..7c83108c 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -376,7 +376,6 @@ def save(self, value: Any) -> None:
def state(self) -> None:
return None
- def collect(self, node_path: Path) -> list[Path]:
+ def collect(self) -> list[Path]:
"""Collect paths defined by the pattern."""
- reference_path = node_path if self.root_dir is None else self.root_dir
- return list(reference_path.glob(self.pattern))
+ return list(self.root_dir.glob(self.pattern)) # type: ignore[union-attr]
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 361375b3..55531696 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -702,6 +702,34 @@ def task_example({node_def}: ...
assert "/*.txt>" in captured
+def test_collect_task_with_delayed_dependencies(runner, tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task
+ from pathlib import Path
+
+ @task(is_ready=lambda *x: True)
+ def task_delayed(
+ paths = DelayedPathNode(pattern="[ab].txt")
+ ) -> Annotated[str, Path("merged.txt")]:
+ path_dict = {path.stem: path for path in paths}
+ return path_dict["a"].read_text() + path_dict["b"].read_text()
+ """
+ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
+
+ result = runner.invoke(cli, ["collect", "--nodes", tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "[ab].txt" in result.output
+
+ tmp_path.joinpath("a.txt").touch()
+ tmp_path.joinpath("b.txt").touch()
+
+ result = runner.invoke(cli, ["collect", "--nodes", tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "a.txt" in result.output
+ assert "b.txt" in result.output
+
+
@pytest.mark.end_to_end()
def test_collect_custom_node_receives_default_name(runner, tmp_path):
source = """
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 4748124d..daf26d9f 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1009,7 +1009,7 @@ def task_example(
assert session.exit_code == ExitCode.OK
assert len(session.tasks) == 1
- assert len(session.tasks[0].produces["return"]) == 2
+ assert len(session.tasks[0].produces["root_path"]) == 2
@pytest.mark.end_to_end()
@@ -1022,7 +1022,7 @@ def test_task_that_depends_on_relative_delayed_path_node(tmp_path):
@task(is_ready=lambda *x: True)
def task_delayed(
paths = DelayedPathNode(pattern="[ab].txt")
- ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
+ ) -> Annotated[str, Path("merged.txt")]:
path_dict = {path.stem: path for path in paths}
return path_dict["a"].read_text() + path_dict["b"].read_text()
"""
From 701b46dda36ab6a356c6797a17d736a687b2fea7 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 13 Nov 2023 18:11:42 +0100
Subject: [PATCH 25/81] Allow tasks to depend on other tasks.
---
docs/source/changes.md | 1 +
src/_pytask/collect.py | 6 +++++-
src/_pytask/dag.py | 23 +++++++++++++++++++++
src/_pytask/execute.py | 6 +++---
src/_pytask/mark/__init__.py | 17 +++++++++++++++
src/_pytask/mark/__init__.pyi | 2 ++
src/_pytask/models.py | 38 +++++++++++++++++++++++++++-------
src/_pytask/task_utils.py | 31 ++++++++++++++++++++++++++++
src/_pytask/traceback.py | 6 +++---
tests/test_task.py | 39 +++++++++++++++++++++++++++++++++++
10 files changed, 155 insertions(+), 14 deletions(-)
diff --git a/docs/source/changes.md b/docs/source/changes.md
index 7ac04f26..315cc9a0 100644
--- a/docs/source/changes.md
+++ b/docs/source/changes.md
@@ -18,6 +18,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
when a product annotation is used with the argument name `produces`. And, allow
`produces` to intake any node.
- {pull}`490` refactors and better tests parsing of dependencies.
+- {pull}`491` allows tasks to depend on other tasks.
## 0.4.2 - 2023-11-8
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 5f35dce9..9a6775de 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -255,6 +255,8 @@ def pytask_collect_task(
)
markers = obj.pytask_meta.markers if hasattr(obj, "pytask_meta") else []
+ collection_id = obj.pytask_meta._id if hasattr(obj, "pytask_meta") else None
+ after = obj.pytask_meta.after if hasattr(obj, "pytask_meta") else []
# Get the underlying function to avoid having different states of the function,
# e.g. due to pytask_meta, in different layers of the wrapping.
@@ -267,6 +269,7 @@ def pytask_collect_task(
depends_on=dependencies,
produces=products,
markers=markers,
+ attributes={"collection_id": collection_id, "after": after},
)
return Task(
base_name=name,
@@ -275,6 +278,7 @@ def pytask_collect_task(
depends_on=dependencies,
produces=products,
markers=markers,
+ attributes={"collection_id": collection_id, "after": after},
)
if isinstance(obj, PTask):
return obj
@@ -295,7 +299,7 @@ def pytask_collect_task(
Please, align the names to ensure reproducibility on case-sensitive file systems \
(often Linux or macOS) or disable this error with 'check_casing_of_paths = false' in \
-your pytask configuration file.
+the pyproject.toml file.
Hint: If parts of the path preceding your project directory are not properly \
formatted, check whether you need to call `.resolve()` on `SRC`, `BLD` or other paths \
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index 07ab23b5..a4d6fd0f 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -21,6 +21,7 @@
from _pytask.database_utils import State
from _pytask.exceptions import ResolvingDependenciesError
from _pytask.mark import Mark
+from _pytask.mark import select_by_after_keyword
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
from _pytask.nodes import PythonNode
@@ -101,6 +102,28 @@ def _add_product(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
return dag
+@hookimpl
+def pytask_dag_modify_dag(session: Session, dag: nx.DiGraph) -> None:
+ """Create dependencies between tasks when using ``@task(after=...)``."""
+ temporary_id_to_task = {
+ task.attributes["collection_id"]: task
+ for task in session.tasks
+ if "collection_id" in task.attributes
+ }
+ for task in session.tasks:
+ after = task.attributes.get("after")
+ if isinstance(after, list):
+ for temporary_id in after:
+ other_task = temporary_id_to_task[temporary_id]
+ dag.add_edge(other_task.signature, task.signature)
+ elif isinstance(after, str):
+ task_signature = task.signature
+ signatures = select_by_after_keyword(session, after)
+ signatures.discard(task_signature)
+ for signature in signatures:
+ dag.add_edge(signature, task_signature)
+
+
@hookimpl
def pytask_dag_select_execution_dag(session: Session, dag: nx.DiGraph) -> None:
"""Select the tasks which need to be executed."""
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index b7dbd5f8..baf86c54 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -125,8 +125,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
"""
for dependency in session.dag.predecessors(task.signature):
- node = session.dag.nodes[dependency]["node"]
- if not node.state():
+ node = session.dag.nodes[dependency].get("node")
+ if isinstance(node, PNode) and not node.state():
msg = f"{task.name!r} requires missing node {node.name!r}."
if IS_FILE_SYSTEM_CASE_SENSITIVE:
msg += (
@@ -138,7 +138,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
# Create directory for product if it does not exist. Maybe this should be a `setup`
# method for the node classes.
for product in session.dag.successors(task.signature):
- node = session.dag.nodes[product]["node"]
+ node = session.dag.nodes[product].get("node")
if isinstance(node, PPathNode):
node.path.parent.mkdir(parents=True, exist_ok=True)
diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py
index f99d1b74..20266a13 100644
--- a/src/_pytask/mark/__init__.py
+++ b/src/_pytask/mark/__init__.py
@@ -39,6 +39,7 @@
"MarkDecorator",
"MarkGenerator",
"ParseError",
+ "select_by_after_keyword",
"select_by_keyword",
"select_by_mark",
]
@@ -168,6 +169,22 @@ def select_by_keyword(session: Session, dag: nx.DiGraph) -> set[str]:
return remaining
+def select_by_after_keyword(session: Session, after: str) -> set[str]:
+ """Select tasks defined by the after keyword."""
+ try:
+ expression = Expression.compile_(after)
+ except ParseError as e:
+ msg = f"Wrong expression passed to 'after': {after}: {e}"
+ raise ValueError(msg) from None
+
+ ancestors: set[str] = set()
+ for task in session.tasks:
+ if after and expression.evaluate(KeywordMatcher.from_task(task)):
+ ancestors.add(task.signature)
+
+ return ancestors
+
+
@define(slots=True)
class MarkMatcher:
"""A matcher for markers which are present.
diff --git a/src/_pytask/mark/__init__.pyi b/src/_pytask/mark/__init__.pyi
index 84d76e41..5518a106 100644
--- a/src/_pytask/mark/__init__.pyi
+++ b/src/_pytask/mark/__init__.pyi
@@ -10,6 +10,7 @@ from _pytask.tree_util import PyTree
from _pytask.session import Session
import networkx as nx
+def select_by_after_keyword(session: Session, after: str) -> set[str]: ...
def select_by_keyword(session: Session, dag: nx.DiGraph) -> set[str]: ...
def select_by_mark(session: Session, dag: nx.DiGraph) -> set[str]: ...
@@ -54,4 +55,5 @@ __all__ = [
"ParseError",
"select_by_keyword",
"select_by_mark",
+ "select_by_after_keyword",
]
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index 247084d4..ff3768fa 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -2,8 +2,11 @@
from __future__ import annotations
from typing import Any
+from typing import Callable
from typing import NamedTuple
from typing import TYPE_CHECKING
+from uuid import UUID
+from uuid import uuid4
from attrs import define
from attrs import field
@@ -16,18 +19,39 @@
@define
class CollectionMetadata:
- """A class for carrying metadata from functions to tasks."""
-
+ """A class for carrying metadata from functions to tasks.
+
+ Attributes
+ ----------
+ after
+ An expression or a task function or a list of task functions that need to be
+ executed before this task can.
+ id_
+ An id for the task if it is part of a parametrization. Otherwise, an automatic
+ id will be generated. See
+ :doc:`this tutorial <../tutorials/repeating_tasks_with_different_inputs>` for
+ more information.
+ kwargs
+ A dictionary containing keyword arguments which are passed to the task when it
+ is executed.
+ markers
+ A list of markers that are attached to the task.
+ name
+ Use it to override the name of the task that is, by default, the name of the
+ callable.
+ produces
+ Definition of products to parse the function returns and store them. See
+ :doc:`this how-to guide <../how_to_guides/using_task_returns>` for more
+ information.
+ """
+
+ after: str | list[Callable[..., Any]] = field(factory=list) # type: ignore[assignment]
id_: str | None = None
- """The id for a single parametrization."""
kwargs: dict[str, Any] = field(factory=dict)
- """Contains kwargs which are necessary for the task function on execution."""
markers: list[Mark] = field(factory=list)
- """Contains the markers of the function."""
name: str | None = None
- """The name of the task function."""
produces: PyTree[Any] | None = None
- """Definition of products to handle returns."""
+ _id: UUID = field(factory=uuid4)
class NodeInfo(NamedTuple):
diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py
index 4ae9d333..25826de9 100644
--- a/src/_pytask/task_utils.py
+++ b/src/_pytask/task_utils.py
@@ -39,6 +39,7 @@
def task(
name: str | None = None,
*,
+ after: str | Callable[..., Any] | list[Callable[..., Any]] | None = None,
id: str | None = None, # noqa: A002
kwargs: dict[Any, Any] | None = None,
produces: PyTree[Any] | None = None,
@@ -55,6 +56,9 @@ def task(
name
Use it to override the name of the task that is, by default, the name of the
callable.
+ after
+ An expression or a task function or a list of task functions that need to be
+ executed before this task can.
id
An id for the task if it is part of a parametrization. Otherwise, an automatic
id will be generated. See
@@ -102,6 +106,7 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
parsed_kwargs = {} if kwargs is None else kwargs
parsed_name = name if isinstance(name, str) else func.__name__
+ parsed_after = _parse_after(after)
if hasattr(unwrapped, "pytask_meta"):
unwrapped.pytask_meta.name = parsed_name
@@ -109,6 +114,7 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
unwrapped.pytask_meta.markers.append(Mark("task", (), {}))
unwrapped.pytask_meta.id_ = id
unwrapped.pytask_meta.produces = produces
+ unwrapped.pytask_meta.after = parsed_after
else:
unwrapped.pytask_meta = CollectionMetadata(
name=parsed_name,
@@ -116,6 +122,7 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
markers=[Mark("task", (), {})],
id_=id,
produces=produces,
+ after=parsed_after,
)
# Store it in the global variable ``COLLECTED_TASKS`` to avoid garbage
@@ -131,6 +138,30 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
return wrapper
+def _parse_after(
+ after: str | Callable[..., Any] | list[Callable[..., Any]] | None
+) -> str | list[Callable[..., Any]]:
+ if not after:
+ return []
+ if isinstance(after, str):
+ return after
+ if callable(after):
+ if not hasattr(after, "pytask_meta"):
+ after.pytask_meta = CollectionMetadata() # type: ignore[attr-defined]
+ return [after.pytask_meta._id] # type: ignore[attr-defined]
+ if isinstance(after, list):
+ new_after = []
+ for func in after:
+ if not hasattr(func, "pytask_meta"):
+ func.pytask_meta = CollectionMetadata() # type: ignore[attr-defined]
+ new_after.append(func.pytask_meta._id) # type: ignore[attr-defined]
+ msg = (
+ "'after' should be an expression string, a task, or a list of class. Got "
+ f"{after}, instead."
+ )
+ raise TypeError(msg)
+
+
def parse_collected_tasks_with_task_marker(
tasks: list[Callable[..., Any]],
) -> dict[str, Callable[..., Any]]:
diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py
index c3f0ea61..d476f39d 100644
--- a/src/_pytask/traceback.py
+++ b/src/_pytask/traceback.py
@@ -47,9 +47,9 @@ class Traceback:
show_locals: ClassVar[bool] = False
suppress: ClassVar[tuple[Path, ...]] = (
- _PLUGGY_DIRECTORY,
- TREE_UTIL_LIB_DIRECTORY,
- _PYTASK_DIRECTORY,
+ # _PLUGGY_DIRECTORY,
+ # TREE_UTIL_LIB_DIRECTORY,
+ # _PYTASK_DIRECTORY,
)
def __rich_console__(
diff --git a/tests/test_task.py b/tests/test_task.py
index 012558a4..5e5dc18b 100644
--- a/tests/test_task.py
+++ b/tests/test_task.py
@@ -615,3 +615,42 @@ def func(path: Annotated[Path, Product]):
assert result.exit_code == ExitCode.COLLECTION_FAILED
assert "Duplicated tasks" in result.output
assert "id=b.txt" in result.output
+
+
+def test_task_will_be_executed_after_another_one_with_string(runner, tmp_path):
+ source = """
+ from pytask import task
+ from pathlib import Path
+ from typing_extensions import Annotated
+
+ @task(after="task_first")
+ def task_second():
+ assert Path("out.txt").exists()
+
+ def task_first() -> Annotated[str, Path("out.txt")]:
+ return "Hello, World!"
+ """
+ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
+
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "2 Succeeded" in result.output
+
+
+def test_task_will_be_executed_after_another_one_with_function(tmp_path):
+ source = """
+ from pytask import task
+ from pathlib import Path
+ from typing_extensions import Annotated
+
+ def task_first() -> Annotated[str, Path("out.txt")]:
+ return "Hello, World!"
+
+ @task(after=task_first)
+ def task_second():
+ assert Path("out.txt").exists()
+ """
+ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
+
+ session = build(paths=tmp_path)
+ assert session.exit_code == ExitCode.OK
From ab9fd8dfbac1db87c96d420458cafd6bc7c6e613 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 13 Nov 2023 18:12:32 +0100
Subject: [PATCH 26/81] Fix.
---
docs/source/changes.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/source/changes.md b/docs/source/changes.md
index 315cc9a0..b2668a3e 100644
--- a/docs/source/changes.md
+++ b/docs/source/changes.md
@@ -18,7 +18,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
when a product annotation is used with the argument name `produces`. And, allow
`produces` to intake any node.
- {pull}`490` refactors and better tests parsing of dependencies.
-- {pull}`491` allows tasks to depend on other tasks.
+- {pull}`493` allows tasks to depend on other tasks.
## 0.4.2 - 2023-11-8
From 8cf360d39fc3fb0de90e2e6f5c99215d060964d8 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 13 Nov 2023 18:18:56 +0100
Subject: [PATCH 27/81] Fix.
---
src/_pytask/traceback.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py
index d476f39d..c3f0ea61 100644
--- a/src/_pytask/traceback.py
+++ b/src/_pytask/traceback.py
@@ -47,9 +47,9 @@ class Traceback:
show_locals: ClassVar[bool] = False
suppress: ClassVar[tuple[Path, ...]] = (
- # _PLUGGY_DIRECTORY,
- # TREE_UTIL_LIB_DIRECTORY,
- # _PYTASK_DIRECTORY,
+ _PLUGGY_DIRECTORY,
+ TREE_UTIL_LIB_DIRECTORY,
+ _PYTASK_DIRECTORY,
)
def __rich_console__(
From 9f61f925442713dd1b6a3fa8a376b9bbb2aa4177 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 13 Nov 2023 18:29:02 +0100
Subject: [PATCH 28/81] Fix paths.
---
tests/test_task.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/test_task.py b/tests/test_task.py
index 5e5dc18b..456c2c20 100644
--- a/tests/test_task.py
+++ b/tests/test_task.py
@@ -625,7 +625,7 @@ def test_task_will_be_executed_after_another_one_with_string(runner, tmp_path):
@task(after="task_first")
def task_second():
- assert Path("out.txt").exists()
+ assert Path(__file__).parent.joinpath("out.txt").exists()
def task_first() -> Annotated[str, Path("out.txt")]:
return "Hello, World!"
@@ -648,7 +648,7 @@ def task_first() -> Annotated[str, Path("out.txt")]:
@task(after=task_first)
def task_second():
- assert Path("out.txt").exists()
+ assert Path(__file__).parent.joinpath("out.txt").exists()
"""
tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
From bd0bb6fd6ca137b8b921a415fea1aa2f4e84ca5d Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Tue, 14 Nov 2023 02:04:14 +0100
Subject: [PATCH 29/81] Fix.
---
.../defining_dependencies_products.md | 32 +++++++++++++++++++
src/_pytask/dag.py | 6 ++--
src/_pytask/execute.py | 6 ++--
tests/test_task.py | 8 +++++
4 files changed, 47 insertions(+), 5 deletions(-)
diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md
index da4d133a..9b50cd6d 100644
--- a/docs/source/tutorials/defining_dependencies_products.md
+++ b/docs/source/tutorials/defining_dependencies_products.md
@@ -410,6 +410,38 @@ def task_fit_model(depends_on, produces):
:::
::::
+## Depending on a task
+
+In some situations you want to say that a task depends on another task without
+specifying the relationship explicitly.
+
+pytask allows you to do that, but you loose features like access to paths which is why
+explicit modeling is always preferred.
+
+There are two modes for it and both use {func}`@task(after=...) <pytask.task>`.
+
+First, you can pass the task function or multiple task functions to the decorator.
+Applied to the tasks from before, we could have written `task_plot_data` as
+
+```python
+@task(after=task_create_random_data)
+def task_plot_data(...):
+ ...
+```
+
+You can also pass a list of task functions.
+
+The second method is to pass an expression, a substring of the name of the dependent
+tasks. Here, we can just pass the function name or a significant part of the function
+name.
+
+```python
+@task(after="random_data")
+def task_plot_data(...):
+ ...
+```
+
+You will learn more about expressions in {doc}`selecting_tasks`.
## References
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index a4d6fd0f..f22b6a03 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -115,13 +115,15 @@ def pytask_dag_modify_dag(session: Session, dag: nx.DiGraph) -> None:
if isinstance(after, list):
for temporary_id in after:
other_task = temporary_id_to_task[temporary_id]
- dag.add_edge(other_task.signature, task.signature)
+ for successor in dag.successors(other_task):
+ dag.add_edge(successor, task.signature)
elif isinstance(after, str):
task_signature = task.signature
signatures = select_by_after_keyword(session, after)
signatures.discard(task_signature)
for signature in signatures:
- dag.add_edge(signature, task_signature)
+ for successor in dag.successors(signature):
+ dag.add_edge(successor, task.signature)
@hookimpl
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index baf86c54..b7dbd5f8 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -125,8 +125,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
"""
for dependency in session.dag.predecessors(task.signature):
- node = session.dag.nodes[dependency].get("node")
- if isinstance(node, PNode) and not node.state():
+ node = session.dag.nodes[dependency]["node"]
+ if not node.state():
msg = f"{task.name!r} requires missing node {node.name!r}."
if IS_FILE_SYSTEM_CASE_SENSITIVE:
msg += (
@@ -138,7 +138,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
# Create directory for product if it does not exist. Maybe this should be a `setup`
# method for the node classes.
for product in session.dag.successors(task.signature):
- node = session.dag.nodes[product].get("node")
+ node = session.dag.nodes[product]["node"]
if isinstance(node, PPathNode):
node.path.parent.mkdir(parents=True, exist_ok=True)
diff --git a/tests/test_task.py b/tests/test_task.py
index 456c2c20..25f57176 100644
--- a/tests/test_task.py
+++ b/tests/test_task.py
@@ -636,6 +636,14 @@ def task_first() -> Annotated[str, Path("out.txt")]:
assert result.exit_code == ExitCode.OK
assert "2 Succeeded" in result.output
+ # Make sure that the dependence does not only apply to the task (and task module),
+ # but also it products.
+ tmp_path.joinpath("out.txt").write_text("Hello, Moon!")
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "1 Succeeded" in result.output
+ assert "1 Skipped because unchanged" in result.output
+
def test_task_will_be_executed_after_another_one_with_function(tmp_path):
source = """
From 413ffbae7bce8f783787ca44bfe3414ebac44cab Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Tue, 14 Nov 2023 02:22:06 +0100
Subject: [PATCH 30/81] fix.
---
src/_pytask/dag.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index f22b6a03..6c998adb 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -115,7 +115,7 @@ def pytask_dag_modify_dag(session: Session, dag: nx.DiGraph) -> None:
if isinstance(after, list):
for temporary_id in after:
other_task = temporary_id_to_task[temporary_id]
- for successor in dag.successors(other_task):
+ for successor in dag.successors(other_task.signature):
dag.add_edge(successor, task.signature)
elif isinstance(after, str):
task_signature = task.signature
From 649aa2436cf74c264d8c87382dbc653e9fa6e6ef Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Tue, 14 Nov 2023 22:01:38 +0100
Subject: [PATCH 31/81] Fix.
---
docs/source/tutorials/defining_dependencies_products.md | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md
index 9b50cd6d..cfb340a4 100644
--- a/docs/source/tutorials/defining_dependencies_products.md
+++ b/docs/source/tutorials/defining_dependencies_products.md
@@ -412,11 +412,11 @@ def task_fit_model(depends_on, produces):
## Depending on a task
-In some situations you want to say that a task depends on another task without
+In some situations you want to define a task depending on another task without
specifying the relationship explicitly.
pytask allows you to do that, but you loose features like access to paths which is why
-explicit modeling is always preferred.
+defining dependencies explicitly is always preferred.
There are two modes for it and both use {func}`@task(after=...) <pytask.task>`.
@@ -431,8 +431,8 @@ def task_plot_data(...):
You can also pass a list of task functions.
-The second method is to pass an expression, a substring of the name of the dependent
-tasks. Here, we can just pass the function name or a significant part of the function
+The second mode is to pass an expression, a substring of the name of the dependent
+tasks. Here, we can pass the function name or a significant part of the function
name.
```python
From 8353c7e097f4f4a4fe8b1edf0d243727aacecae8 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Tue, 14 Nov 2023 23:01:40 +0100
Subject: [PATCH 32/81] fix.
---
src/_pytask/delayed.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 94ae82e0..febf49b3 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -1,15 +1,16 @@
from __future__ import annotations
-from pathlib import Path
from typing import TYPE_CHECKING
from _pytask.config import hookimpl
-from _pytask.models import NodeInfo
-from _pytask.node_protocols import PDelayedNode, PTask
+from _pytask.node_protocols import PDelayedNode
+from _pytask.node_protocols import PTask
from _pytask.nodes import DelayedPathNode
from _pytask.outcomes import CollectionOutcome
if TYPE_CHECKING:
+ from _pytask.models import NodeInfo
+ from pathlib import Path
from _pytask.session import Session
From e1cd75a217b088fa41e505863d4ec48655575a32 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 15 Nov 2023 14:40:48 +0100
Subject: [PATCH 33/81] Fix.
---
src/_pytask/collect.py | 28 ++++----
src/_pytask/collect_utils.py | 44 +-----------
src/_pytask/delayed.py | 126 ++++++++++++++++++++--------------
src/_pytask/execute.py | 79 +--------------------
src/_pytask/hookspecs.py | 8 ---
src/_pytask/models.py | 3 -
src/_pytask/task_utils.py | 8 +--
tests/test_collect.py | 31 ---------
tests/test_collect_command.py | 5 +-
tests/test_execute.py | 60 ++++++++--------
tests/test_task_utils.py | 1 -
11 files changed, 124 insertions(+), 269 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index da224673..72d21146 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -25,10 +25,11 @@
from _pytask.mark import MarkGenerator
from _pytask.mark_utils import get_all_marks
from _pytask.mark_utils import has_mark
-from _pytask.models import DelayedTask
+from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
+from _pytask.nodes import DelayedPathNode
from _pytask.nodes import PathNode
from _pytask.nodes import PythonNode
from _pytask.nodes import Task
@@ -255,18 +256,6 @@ def pytask_collect_task(
)
raise ValueError(msg)
- # Collect delayed tasks separately and exit.
- if hasattr(obj, "pytask_meta") and obj.pytask_meta.is_ready is not None:
- try:
- is_ready = obj.pytask_meta.is_ready()
- except Exception as e: # noqa: BLE001
- msg = "The function for the 'is_ready' condition failed."
- raise ValueError(msg) from e
-
- if not is_ready:
- session.delayed_tasks.append(DelayedTask(path=path, name=name, obj=obj))
- return None
-
path_nodes = Path.cwd() if path is None else path.parent
# Collect dependencies and products.
@@ -313,9 +302,9 @@ def pytask_collect_task(
@hookimpl(trylast=True)
-def pytask_collect_node( # noqa: C901
+def pytask_collect_node( # noqa: C901, PLR0912
session: Session, path: Path, node_info: NodeInfo
-) -> PNode:
+) -> PNode | PDelayedNode:
"""Collect a node of a task as a :class:`pytask.PNode`.
Strings are assumed to be paths. This might be a strict assumption, but since this
@@ -335,6 +324,15 @@ def pytask_collect_node( # noqa: C901
"""
node = node_info.value
+ if isinstance(node, DelayedPathNode):
+ if node.root_dir is None:
+ node.root_dir = path
+ if not node.name:
+ node.name = node.root_dir.joinpath(node.pattern).as_posix()
+
+ if isinstance(node, PDelayedNode):
+ return node
+
if isinstance(node, PythonNode):
node.node_info = node_info
if not node.name:
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index f609dc0d..80c3af18 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -262,10 +262,6 @@ def parse_dependencies_from_task_function(
dependencies["depends_on"] = nodes
task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {}
- allow_delayed = (
- obj.pytask_meta.is_ready is not None if hasattr(obj, "pytask_meta") else False
- )
- is_ready = obj.pytask_meta.is_ready() if allow_delayed else False
signature_defaults = parse_keyword_arguments_from_signature_defaults(obj)
kwargs = {**signature_defaults, **task_kwargs}
kwargs.pop("produces", None)
@@ -297,19 +293,8 @@ def parse_dependencies_from_task_function(
if parameter_name in parameters_with_product_annot:
continue
- # Check for delayed nodes.
- has_delayed_nodes = any(isinstance(i, PDelayedNode) for i in tree_leaves(value))
- if has_delayed_nodes and not allow_delayed:
- msg = (
- f"The task {task_name!r} is not marked as a delayed task, but it "
- "depends on a delayed node. Use '@task(is_ready=...) to create a "
- "delayed task."
- )
- raise ValueError(msg)
-
nodes = _collect_delayed_nodes_and_nodes(
collect_dependency,
- is_ready,
session,
node_path,
task_name,
@@ -447,7 +432,6 @@ def parse_products_from_task_function( # noqa: C901
)
collected_products = _collect_delayed_nodes_and_nodes(
_collect_product,
- False,
session,
node_path,
task_name,
@@ -462,7 +446,6 @@ def parse_products_from_task_function( # noqa: C901
has_task_decorator = True
collected_products = _collect_delayed_nodes_and_nodes(
_collect_product,
- False,
session,
node_path,
task_name,
@@ -491,7 +474,6 @@ def parse_products_from_task_function( # noqa: C901
def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
collection_func: Callable[..., Any],
- is_ready: bool,
session: Session,
node_path: Path,
task_name: str,
@@ -499,26 +481,6 @@ def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
parameter_name: str,
value: Any,
) -> PyTree[PDelayedNode | PNode]:
- nodes = tree_map_with_path(
- lambda p, x: session.hook.pytask_collect_delayed_node(
- session=session,
- path=node_path,
- node_info=NodeInfo(
- arg_name=parameter_name,
- path=p,
- value=x,
- task_path=task_path,
- task_name=task_name,
- ),
- )
- if isinstance(x, PDelayedNode)
- else x,
- value,
- )
- if is_ready:
- nodes = tree_map(
- lambda x: x.collect() if isinstance(x, PDelayedNode) else x, nodes
- )
return tree_map_with_path(
lambda p, x: collection_func(
session,
@@ -531,10 +493,8 @@ def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
task_path=task_path,
task_name=task_name,
),
- )
- if not isinstance(x, PDelayedNode)
- else x,
- nodes,
+ ),
+ value,
)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index febf49b3..4f363c5a 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -1,70 +1,96 @@
from __future__ import annotations
+from pathlib import Path
+from typing import Any
from typing import TYPE_CHECKING
+from _pytask.collect_utils import collect_dependency
from _pytask.config import hookimpl
+from _pytask.dag_utils import TopologicalSorter
+from _pytask.models import NodeInfo
from _pytask.node_protocols import PDelayedNode
+from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
-from _pytask.nodes import DelayedPathNode
-from _pytask.outcomes import CollectionOutcome
+from _pytask.node_protocols import PTaskWithPath
+from _pytask.nodes import Task
+from _pytask.tree_util import PyTree
+from _pytask.tree_util import tree_map_with_path
if TYPE_CHECKING:
- from _pytask.models import NodeInfo
- from pathlib import Path
from _pytask.session import Session
-@hookimpl(trylast=True)
-def pytask_collect_delayed_node(path: Path, node_info: NodeInfo) -> PDelayedNode:
- """Collect a delayed node."""
- node = node_info.value
- if isinstance(node, DelayedPathNode):
- if node.root_dir is None:
- node.root_dir = path
- if not node.name:
- node.name = node.root_dir.joinpath(node.pattern).as_posix()
- return node
+_TASKS_WITH_DELAYED_NODES = set()
@hookimpl
-def pytask_execute_collect_delayed_tasks(session: Session) -> None:
- """Collect tasks that are delayed."""
- # Separate ready and delayed tasks.
- still_delayed_tasks = []
- ready_tasks = []
-
- for delayed_task in session.delayed_tasks:
- if delayed_task.obj.pytask_meta.is_ready():
- ready_tasks.append(delayed_task)
- else:
- still_delayed_tasks.append(delayed_task)
-
- session.delayed_tasks = still_delayed_tasks
-
- if not ready_tasks:
- return
-
- # Collect tasks that are now ready.
- new_collection_reports = []
- for ready_task in ready_tasks:
- report = session.hook.pytask_collect_task_protocol(
- session=session,
- reports=session.collection_reports,
- path=ready_task.path,
- name=ready_task.name,
- obj=ready_task.obj,
+def pytask_execute_task_setup(session: Session, task: PTask) -> None:
+ """Collect delayed nodes and parse them."""
+ task.depends_on = tree_map_with_path( # type: ignore[assignment]
+ lambda p, x: _collect_delayed_nodes(session, task, x, p), task.depends_on
+ )
+ if task.signature in _TASKS_WITH_DELAYED_NODES:
+ # Recreate the DAG.
+ session.hook.pytask_dag(session=session)
+ # Update scheduler.
+ session.scheduler = TopologicalSorter.from_dag_and_sorter(
+ dag=session.dag, sorter=session.scheduler
)
- if report is not None:
- new_collection_reports.append(report)
-
- # What to do with failed tasks? Can we just add them to executionreports.
- # Add new tasks.
- session.tasks.extend(
- i.node
- for i in session.collection_reports
- if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, PTask)
+@hookimpl
+def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
+ """Check if nodes are produced by a task."""
+ # Replace delayed nodes with their actually resolved nodes.
+ task.produces = tree_map_with_path( # type: ignore[assignment]
+ lambda p, x: _collect_delayed_nodes(session, task, x, p), task.produces
)
- session.hook.pytask_dag(session=session)
+ if task.signature in _TASKS_WITH_DELAYED_NODES:
+ # Recreate the DAG.
+ session.hook.pytask_dag(session=session)
+ # Update scheduler.
+ session.scheduler = TopologicalSorter.from_dag_and_sorter(
+ dag=session.dag, sorter=session.scheduler
+ )
+
+
+def _collect_delayed_nodes(
+ session: Session, task: PTask, node: Any, path: tuple[Any, ...]
+) -> PyTree[PNode]:
+ """Collect delayed nodes.
+
+ 1. Call the :meth:`pytask.PDelayedNode.collect` to receive the raw nodes.
+ 2. Collect the raw nodes as usual.
+
+ """
+ if not isinstance(node, PDelayedNode):
+ return node
+
+ # Add task to register to update the DAG after the task is executed.
+ _TASKS_WITH_DELAYED_NODES.add(task.signature)
+
+ # Collect delayed nodes and receive raw nodes.
+ delayed_nodes = node.collect()
+
+ # Collect raw nodes.
+ node_path = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd()
+ task_name = task.base_name if isinstance(task, Task) else task.name
+ task_path = task.path if isinstance(task, PTaskWithPath) else None
+ arg_name, *rest_path = path
+
+ return tree_map_with_path(
+ lambda p, x: collect_dependency(
+ session,
+ node_path,
+ task_name,
+ NodeInfo(
+ arg_name=arg_name,
+ path=(*rest_path, *p),
+ value=x,
+ task_path=task_path,
+ task_name=task_name,
+ ),
+ ),
+ delayed_nodes,
+ )
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index bbe8068a..016fe5dc 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -4,11 +4,9 @@
import inspect
import sys
import time
-from pathlib import Path
from typing import Any
from typing import TYPE_CHECKING
-from _pytask.collect_utils import collect_dependency
from _pytask.config import hookimpl
from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE
from _pytask.console import console
@@ -27,14 +25,10 @@
from _pytask.exceptions import NodeNotFoundError
from _pytask.mark import Mark
from _pytask.mark_utils import has_mark
-from _pytask.models import NodeInfo
from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
-from _pytask.node_protocols import PTaskWithPath
-from _pytask.nodes import Task
-from _pytask.outcomes import CollectionOutcome
from _pytask.outcomes import count_outcomes
from _pytask.outcomes import Exit
from _pytask.outcomes import SkippedUnchanged
@@ -42,10 +36,8 @@
from _pytask.outcomes import WouldBeExecuted
from _pytask.reports import ExecutionReport
from _pytask.traceback import remove_traceback_from_exc_info
-from _pytask.tree_util import PyTree
from _pytask.tree_util import tree_leaves
from _pytask.tree_util import tree_map
-from _pytask.tree_util import tree_map_with_path
from _pytask.tree_util import tree_structure
from rich.text import Text
@@ -223,78 +215,9 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
return True
-def _collect_delayed_nodes(
- session: Session, task: PTask, node: Any, path: tuple[Any, ...]
-) -> PyTree[PNode]:
- """Collect delayed nodes."""
- if not isinstance(node, PDelayedNode):
- return node
-
- node_path = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd()
- task_name = task.base_name if isinstance(task, Task) else task.name
- task_path = task.path if isinstance(task, PTaskWithPath) else None
- arg_name, *rest_path = path
-
- delayed_nodes = node.collect()
- return tree_map_with_path(
- lambda p, x: collect_dependency(
- session,
- node_path,
- task_name,
- NodeInfo(
- arg_name=arg_name,
- path=(*rest_path, *p),
- value=x,
- task_path=task_path,
- task_name=task_name,
- ),
- ),
- delayed_nodes,
- )
-
-
-@hookimpl
+@hookimpl(trylast=True)
def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
"""Check if nodes are produced by a task."""
- # Replace delayed nodes with their actually resolved nodes.
- has_delayed_nodes = any(
- isinstance(node, PDelayedNode) for node in tree_leaves(task.produces)
- )
- if has_delayed_nodes:
- # Collect delayed nodes
- task.produces = tree_map_with_path( # type: ignore[assignment]
- lambda p, x: _collect_delayed_nodes(session, task, x, p), task.produces
- )
- # Collect delayed tasks.
- new_collection_reports = []
- still_delayed_tasks = []
- for delayed_task in session.delayed_tasks:
- if delayed_task.obj.pytask_meta.is_ready():
- report = session.hook.pytask_collect_task_protocol(
- session=session,
- reports=session.collection_reports,
- path=delayed_task.path,
- name=delayed_task.name,
- obj=delayed_task.obj,
- )
- new_collection_reports.append(report)
- else:
- still_delayed_tasks.append(delayed_task)
- session.delayed_tasks = still_delayed_tasks
- session.collection_reports.extend(new_collection_reports)
- session.tasks.extend(
- i.node
- for i in new_collection_reports
- if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, PTask)
- )
-
- # Recreate the DAG.
- session.hook.pytask_dag(session=session)
- # Update scheduler.
- session.scheduler = TopologicalSorter.from_dag_and_sorter(
- dag=session.dag, sorter=session.scheduler
- )
-
missing_nodes = [node for node in tree_leaves(task.produces) if not node.state()]
if missing_nodes:
paths = session.config["paths"]
diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py
index 0ec036b7..5680feff 100644
--- a/src/_pytask/hookspecs.py
+++ b/src/_pytask/hookspecs.py
@@ -13,7 +13,6 @@
if TYPE_CHECKING:
- from _pytask.node_protocols import PDelayedNode
from _pytask.models import NodeInfo
from _pytask.node_protocols import PNode
import click
@@ -194,13 +193,6 @@ def pytask_collect_task_teardown(session: Session, task: PTask) -> None:
"""
-@hookspec(firstresult=True)
-def pytask_collect_delayed_node(
- session: Session, path: Path, node_info: NodeInfo
-) -> PDelayedNode | None:
- """Collect a delayed node."""
-
-
@hookspec(firstresult=True)
def pytask_collect_node(
session: Session, path: Path, node_info: NodeInfo
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index bd85ef9a..f30e9f48 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -31,8 +31,6 @@ class CollectionMetadata:
id will be generated. See
:doc:`this tutorial <../tutorials/repeating_tasks_with_different_inputs>` for
more information.
- is_ready
- A callable that indicates whether a delayed task is ready.
kwargs
A dictionary containing keyword arguments which are passed to the task when it
is executed.
@@ -49,7 +47,6 @@ class CollectionMetadata:
after: str | list[Callable[..., Any]] = field(factory=list)
id_: str | None = None
- is_ready: Callable[..., bool] | None = None
kwargs: dict[str, Any] = field(factory=dict)
markers: list[Mark] = field(factory=list)
name: str | None = None
diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py
index 8f6ab635..947c47d1 100644
--- a/src/_pytask/task_utils.py
+++ b/src/_pytask/task_utils.py
@@ -36,12 +36,11 @@
"""
-def task( # noqa: PLR0913
+def task(
name: str | None = None,
*,
after: str | Callable[..., Any] | list[Callable[..., Any]] | None = None,
id: str | None = None, # noqa: A002
- is_ready: Callable[..., bool] | None = None,
kwargs: dict[Any, Any] | None = None,
produces: PyTree[Any] | None = None,
) -> Callable[..., Callable[..., Any]]:
@@ -65,9 +64,6 @@ def task( # noqa: PLR0913
id will be generated. See
:doc:`this tutorial <../tutorials/repeating_tasks_with_different_inputs>` for
more information.
- is_ready
- A callable that indicates when a delayed task is ready. The value is ``None``
- for a normal task.
kwargs
A dictionary containing keyword arguments which are passed to the task when it
is executed.
@@ -114,7 +110,6 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
if hasattr(unwrapped, "pytask_meta"):
unwrapped.pytask_meta.id_ = id
- unwrapped.pytask_meta.is_ready = is_ready
unwrapped.pytask_meta.kwargs = parsed_kwargs
unwrapped.pytask_meta.markers.append(Mark("task", (), {}))
unwrapped.pytask_meta.name = parsed_name
@@ -123,7 +118,6 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
else:
unwrapped.pytask_meta = CollectionMetadata(
id_=id,
- is_ready=is_ready,
kwargs=parsed_kwargs,
markers=[Mark("task", (), {})],
name=parsed_name,
diff --git a/tests/test_collect.py b/tests/test_collect.py
index dbdc8d75..78902669 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -581,37 +581,6 @@ def task_example(path: Annotated[Path, Path("file.txt"), Product]) -> None: ...
assert "is defined twice" in result.output
-@pytest.mark.end_to_end()
-def test_task_missing_is_ready_cannot_depend_on_delayed_node(runner, tmp_path):
- source = """
- from pytask import DelayedPathNode
-
- def task_example(a = DelayedPathNode(pattern="*.txt")): ...
- """
- tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
-
- result = runner.invoke(cli, [tmp_path.as_posix()])
- assert result.exit_code == ExitCode.COLLECTION_FAILED
- assert "The task 'task_example' is not marked" in result.output
-
-
-@pytest.mark.end_to_end()
-def test_gracefully_fail_with_failing_is_ready_condition(runner, tmp_path):
- source = """
- from pytask import task
-
- def raise_(): raise Exception("ERROR")
-
- @task(is_ready=raise_)
- def task_example(): ...
- """
- tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
-
- result = runner.invoke(cli, [tmp_path.as_posix()])
- assert result.exit_code == ExitCode.COLLECTION_FAILED
- assert "The function for the 'is_ready' condition failed." in result.output
-
-
@pytest.mark.parametrize(
"node",
[
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 55531696..e39dc72a 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -705,11 +705,10 @@ def task_example({node_def}: ...
def test_collect_task_with_delayed_dependencies(runner, tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task
+ from pytask import DelayedPathNode
from pathlib import Path
- @task(is_ready=lambda *x: True)
- def task_delayed(
+ def task_example(
paths = DelayedPathNode(pattern="[ab].txt")
) -> Annotated[str, Path("merged.txt")]:
path_dict = {path.stem: path for path in paths}
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 1a32c335..1c746487 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -969,6 +969,30 @@ def task_example(a = PythonNode(value={"a": 1}, hash=True)):
assert "TypeError: unhashable type: 'dict'" in result.output
+def test_task_is_not_reexecuted(runner, tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pathlib import Path
+
+ def task_first() -> Annotated[str, Path("out.txt")]:
+ return "Hello, World!"
+
+ def task_second(path = Path("out.txt")) -> Annotated[str, Path("copy.txt")]:
+ return path.read_text()
+ """
+ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
+
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "2 Succeeded" in result.output
+
+ tmp_path.joinpath("out.txt").write_text("Changed text.")
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "1 Succeeded" in result.output
+ assert "1 Skipped because unchanged" in result.output
+
+
@pytest.mark.end_to_end()
def test_task_that_produces_delayed_path_node(tmp_path):
source = """
@@ -995,11 +1019,10 @@ def task_example(
def test_task_that_depends_on_relative_delayed_path_node(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task
+ from pytask import DelayedPathNode
from pathlib import Path
- @task(is_ready=lambda *x: True)
- def task_delayed(
+ def task_example(
paths = DelayedPathNode(pattern="[ab].txt")
) -> Annotated[str, Path("merged.txt")]:
path_dict = {path.stem: path for path in paths}
@@ -1020,13 +1043,12 @@ def task_delayed(
def test_task_that_depends_on_delayed_path_node_with_root_dir(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task
+ from pytask import DelayedPathNode
from pathlib import Path
root_dir = Path(__file__).parent / "subfolder"
- @task(is_ready=lambda *x: True)
- def task_delayed(
+ def task_example(
paths = DelayedPathNode(root_dir=root_dir, pattern="[ab].txt")
) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
path_dict = {path.stem: path for path in paths}
@@ -1056,7 +1078,7 @@ def task_produces() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
- @task(is_ready=lambda *x: Path(__file__).parent.joinpath("a.txt").exists())
+ @task(after=task_produces)
def task_depends(
paths = DelayedPathNode(pattern="[ab].txt")
) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
@@ -1071,27 +1093,3 @@ def task_depends(
assert len(session.tasks) == 2
assert len(session.tasks[0].produces["return"]) == 2
assert len(session.tasks[1].depends_on["paths"]) == 2
-
-
-def test_task_is_not_reexecuted(runner, tmp_path):
- source = """
- from typing_extensions import Annotated
- from pathlib import Path
-
- def task_first() -> Annotated[str, Path("out.txt")]:
- return "Hello, World!"
-
- def task_second(path = Path("out.txt")) -> Annotated[str, Path("copy.txt")]:
- return path.read_text()
- """
- tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
-
- result = runner.invoke(cli, [tmp_path.as_posix()])
- assert result.exit_code == ExitCode.OK
- assert "2 Succeeded" in result.output
-
- tmp_path.joinpath("out.txt").write_text("Changed text.")
- result = runner.invoke(cli, [tmp_path.as_posix()])
- assert result.exit_code == ExitCode.OK
- assert "1 Succeeded" in result.output
- assert "1 Skipped because unchanged" in result.output
diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py
index 2f9d354f..aa5cde7f 100644
--- a/tests/test_task_utils.py
+++ b/tests/test_task_utils.py
@@ -67,7 +67,6 @@ def task_example():
...
assert task_example.pytask_meta.id_ is None
- assert task_example.pytask_meta.is_ready is None
assert task_example.pytask_meta.kwargs == {}
assert task_example.pytask_meta.markers == [Mark("task", (), {})]
assert task_example.pytask_meta.name == "task_example"
From 137144cd08f05dfa27bd3e42a724c77e91e1912a Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 15 Nov 2023 15:07:27 +0100
Subject: [PATCH 34/81] Fix.
---
src/_pytask/collect.py | 10 ++++++++--
tests/test_collect_command.py | 8 --------
2 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 72d21146..bce8ba2e 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -327,8 +327,14 @@ def pytask_collect_node( # noqa: C901, PLR0912
if isinstance(node, DelayedPathNode):
if node.root_dir is None:
node.root_dir = path
- if not node.name:
- node.name = node.root_dir.joinpath(node.pattern).as_posix()
+ if (
+ not node.name
+ or node.name == node.root_dir.joinpath(node.pattern).as_posix()
+ ):
+ short_root_dir = shorten_path(
+ node.root_dir, session.config["paths"] or (session.config["root"],)
+ )
+ node.name = Path(short_root_dir, node.pattern).as_posix()
if isinstance(node, PDelayedNode):
return node
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index e39dc72a..b543e181 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -720,14 +720,6 @@ def task_example(
assert result.exit_code == ExitCode.OK
assert "[ab].txt" in result.output
- tmp_path.joinpath("a.txt").touch()
- tmp_path.joinpath("b.txt").touch()
-
- result = runner.invoke(cli, ["collect", "--nodes", tmp_path.as_posix()])
- assert result.exit_code == ExitCode.OK
- assert "a.txt" in result.output
- assert "b.txt" in result.output
-
@pytest.mark.end_to_end()
def test_collect_custom_node_receives_default_name(runner, tmp_path):
From 4903307734c7468a85c2ee7903968b849b7f895d Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 15 Nov 2023 15:18:49 +0100
Subject: [PATCH 35/81] Fix.
---
tests/test_collect_command.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index b543e181..5389d5ad 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -666,7 +666,7 @@ def task_example() -> Annotated[Dict[str, str], nodes]:
@pytest.mark.parametrize(
"node_def",
[
- "paths: Annotated[list[Path], DelayedPathNode(pattern='*.txt'), Product])",
+ "paths: Annotated[List[Path], DelayedPathNode(pattern='*.txt'), Product])",
"produces=DelayedPathNode(pattern='*.txt'))",
") -> Annotated[None, DelayedPathNode(pattern='*.txt')]",
],
@@ -674,7 +674,7 @@ def task_example() -> Annotated[Dict[str, str], nodes]:
def test_collect_task_with_delayed_path_node_as_product(runner, tmp_path, node_def):
source = f"""
from pytask import DelayedPathNode, Product
- from typing_extensions import Annotated
+ from typing_extensions import Annotated, List
from pathlib import Path
def task_example({node_def}: ...
From 92e99900b2331b76ec7eba00c172ff90ebf5cb41 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 15 Nov 2023 16:18:54 +0100
Subject: [PATCH 36/81] Raise errors gracefully when recreating the DAG.
---
src/_pytask/dag.py | 7 +++++
src/_pytask/delayed.py | 31 +++++++++++--------
src/_pytask/nodes.py | 11 ++++---
src/_pytask/report.py | 70 ------------------------------------------
tests/test_execute.py | 70 +++++++++++++++++++++++++++++++++++++++++-
5 files changed, 100 insertions(+), 89 deletions(-)
delete mode 100644 src/_pytask/report.py
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index 1278b8aa..85b480c8 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -16,6 +16,7 @@
from _pytask.console import TASK_ICON
from _pytask.exceptions import ResolvingDependenciesError
from _pytask.mark import select_by_after_keyword
+from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
from _pytask.nodes import PythonNode
@@ -56,6 +57,9 @@ def pytask_dag_create_dag(session: Session, tasks: list[PTask]) -> nx.DiGraph:
def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
"""Add a dependency to the DAG."""
+ if isinstance(node, PDelayedNode):
+ return
+
dag.add_node(node.signature, node=node)
dag.add_edge(node.signature, task.signature)
@@ -67,6 +71,9 @@ def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
def _add_product(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
"""Add a product to the DAG."""
+ if isinstance(node, PDelayedNode):
+ return
+
dag.add_node(node.signature, node=node)
dag.add_edge(task.signature, node.signature)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 4f363c5a..07f83095 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -1,18 +1,19 @@
from __future__ import annotations
+import sys
from pathlib import Path
from typing import Any
from typing import TYPE_CHECKING
from _pytask.collect_utils import collect_dependency
from _pytask.config import hookimpl
-from _pytask.dag_utils import TopologicalSorter
from _pytask.models import NodeInfo
from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
from _pytask.nodes import Task
+from _pytask.reports import ExecutionReport
from _pytask.tree_util import PyTree
from _pytask.tree_util import tree_map_with_path
@@ -30,12 +31,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
lambda p, x: _collect_delayed_nodes(session, task, x, p), task.depends_on
)
if task.signature in _TASKS_WITH_DELAYED_NODES:
- # Recreate the DAG.
- session.hook.pytask_dag(session=session)
- # Update scheduler.
- session.scheduler = TopologicalSorter.from_dag_and_sorter(
- dag=session.dag, sorter=session.scheduler
- )
+ _recreate_dag(session, task)
@hookimpl
@@ -47,12 +43,7 @@ def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
)
if task.signature in _TASKS_WITH_DELAYED_NODES:
- # Recreate the DAG.
- session.hook.pytask_dag(session=session)
- # Update scheduler.
- session.scheduler = TopologicalSorter.from_dag_and_sorter(
- dag=session.dag, sorter=session.scheduler
- )
+ _recreate_dag(session, task)
def _collect_delayed_nodes(
@@ -94,3 +85,17 @@ def _collect_delayed_nodes(
),
delayed_nodes,
)
+
+
+def _recreate_dag(session: Session, task: PTask) -> None:
+ """Recreate the DAG."""
+ try:
+ session.dag = session.hook.pytask_dag_create_dag(
+ session=session, tasks=session.tasks
+ )
+ session.hook.pytask_dag_modify_dag(session=session, dag=session.dag)
+
+ except Exception: # noqa: BLE001
+ report = ExecutionReport.from_task_and_exception(task, sys.exc_info())
+ session.execution_reports.append(report)
+ session.should_stop = True
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 7c83108c..92a61e05 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -361,20 +361,21 @@ class DelayedPathNode(PNode):
@property
def signature(self) -> str:
"""The unique signature of the node."""
+ raise NotImplementedError
raw_key = "".join(str(hash_value(arg)) for arg in (self.root_dir, self.pattern))
return hashlib.sha256(raw_key.encode()).hexdigest()
- def load(self, is_product: bool = False) -> Path | tuple[Path, str]:
- assert self.root_dir
+ def load(self, is_product: bool = False) -> Path:
if is_product:
- return self.root_dir
- return self.root_dir, self.pattern
+ return self.root_dir # type: ignore[return-value]
+ raise NotImplementedError
def save(self, value: Any) -> None:
raise NotImplementedError
def state(self) -> None:
- return None
+ raise NotImplementedError
+ return "0"
def collect(self) -> list[Path]:
"""Collect paths defined by the pattern."""
diff --git a/src/_pytask/report.py b/src/_pytask/report.py
deleted file mode 100644
index 62fcbbdb..00000000
--- a/src/_pytask/report.py
+++ /dev/null
@@ -1,70 +0,0 @@
-"""Contains everything related to reports."""
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from _pytask.outcomes import CollectionOutcome
-from _pytask.outcomes import TaskOutcome
-from _pytask.traceback import OptionalExceptionInfo
-from _pytask.traceback import remove_internal_traceback_frames_from_exc_info
-from attrs import define
-from attrs import field
-
-
-if TYPE_CHECKING:
- from _pytask.node_protocols import PTask
- from _pytask.node_protocols import MetaNode
-
-
-@define
-class CollectionReport:
- """A collection report for a task."""
-
- outcome: CollectionOutcome
- node: MetaNode | None = None
- exc_info: OptionalExceptionInfo | None = None
-
- @classmethod
- def from_exception(
- cls: type[CollectionReport],
- outcome: CollectionOutcome,
- exc_info: OptionalExceptionInfo,
- node: MetaNode | None = None,
- ) -> CollectionReport:
- exc_info = remove_internal_traceback_frames_from_exc_info(exc_info)
- return cls(outcome=outcome, node=node, exc_info=exc_info)
-
-
-@define
-class DagReport:
- """A report for an error during the creation of the DAG."""
-
- exc_info: OptionalExceptionInfo
-
- @classmethod
- def from_exception(cls, exc_info: OptionalExceptionInfo) -> DagReport:
- exc_info = remove_internal_traceback_frames_from_exc_info(exc_info)
- return cls(exc_info)
-
-
-@define
-class ExecutionReport:
- """A report for an executed task."""
-
- task: PTask
- outcome: TaskOutcome
- exc_info: OptionalExceptionInfo | None = None
- sections: list[tuple[str, str, str]] = field(factory=list)
-
- @classmethod
- def from_task_and_exception(
- cls, task: PTask, exc_info: OptionalExceptionInfo
- ) -> ExecutionReport:
- """Create a report from a task and an exception."""
- exc_info = remove_internal_traceback_frames_from_exc_info(exc_info)
- return cls(task, TaskOutcome.FAIL, exc_info, task.report_sections)
-
- @classmethod
- def from_task(cls, task: PTask) -> ExecutionReport:
- """Create a report from a task."""
- return cls(task, TaskOutcome.SUCCESS, None, task.report_sections)
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 1c746487..87f03563 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1014,6 +1014,10 @@ def task_example(
assert len(session.tasks) == 1
assert len(session.tasks[0].produces["root_path"]) == 2
+ # Rexecution does skip the task.
+ session = build(paths=tmp_path)
+ assert session.execution_reports[0].outcome == TaskOutcome.SKIP_UNCHANGED
+
@pytest.mark.end_to_end()
def test_task_that_depends_on_relative_delayed_path_node(tmp_path):
@@ -1073,6 +1077,35 @@ def test_task_that_depends_on_delayed_task(tmp_path):
from pytask import DelayedPathNode, task
from pathlib import Path
+ def task_produces() -> Annotated[None, DelayedPathNode(pattern="[ab].txt")]:
+ path = Path(__file__).parent
+ path.joinpath("a.txt").write_text("Hello, ")
+ path.joinpath("b.txt").write_text("World!")
+
+ @task(after=task_produces)
+ def task_depends(
+ paths = DelayedPathNode(pattern="[ab].txt")
+ ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
+ path_dict = {path.stem: path for path in paths}
+ return path_dict["a"].read_text() + path_dict["b"].read_text()
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ session = build(paths=tmp_path)
+
+ assert session.exit_code == ExitCode.OK
+ assert len(session.tasks) == 2
+ assert len(session.tasks[0].produces["return"]) == 2
+ assert len(session.tasks[1].depends_on["paths"]) == 2
+
+
+@pytest.mark.end_to_end()
+def test_gracefully_fail_when_dag_raises_error(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task
+ from pathlib import Path
+
def task_produces() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
@@ -1084,7 +1117,42 @@ def task_depends(
) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
path_dict = {path.stem: path for path in paths}
return path_dict["a"].read_text() + path_dict["b"].read_text()
- """
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ session = build(paths=tmp_path)
+ assert session.exit_code == ExitCode.OK
+
+ session = build(paths=tmp_path)
+ assert session.exit_code == ExitCode.FAILED
+ assert session.execution_reports[1].outcome == TaskOutcome.FAIL
+
+
+@pytest.mark.end_to_end()
+def test_delayed_task_generation(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task
+ from pathlib import Path
+
+ def task_produces() -> Annotated[None, DelayedPathNode(pattern="[ab].txt")]:
+ path = Path(__file__).parent
+ path.joinpath("a.txt").write_text("Hello, ")
+ path.joinpath("b.txt").write_text("World!")
+
+ @task(after=task_produces)
+ def task_depends(
+ paths = DelayedPathNode(pattern="[ab].txt")
+ ) -> ...:
+ for path in paths:
+
+ def task_copy(
+ path: Path = path
+ ) -> Annotated[str, path.with_name(path.stem + "-copy.txt")]:
+ return path.read_text()
+
+ yield task_copy
+ """
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
session = build(paths=tmp_path)
From 71dd73862be2d2a8081f352c5b6905ed842934c1 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 15 Nov 2023 16:28:40 +0100
Subject: [PATCH 37/81] Fix.
---
src/_pytask/nodes.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 92a61e05..94d51ae7 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -375,7 +375,6 @@ def save(self, value: Any) -> None:
def state(self) -> None:
raise NotImplementedError
- return "0"
def collect(self) -> list[Path]:
"""Collect paths defined by the pattern."""
From e0789279bede6f8e303b5d2b5626d95af102fded Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 15 Nov 2023 17:59:29 +0100
Subject: [PATCH 38/81] temp.
---
src/_pytask/collect.py | 13 +++++++++++--
src/_pytask/delayed.py | 40 +++++++++++++++++++++++++++++++++++++++
src/_pytask/models.py | 1 +
src/_pytask/task_utils.py | 11 ++++++++---
4 files changed, 60 insertions(+), 5 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index bce8ba2e..64a5213a 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -269,6 +269,7 @@ def pytask_collect_task(
markers = get_all_marks(obj)
collection_id = obj.pytask_meta._id if hasattr(obj, "pytask_meta") else None
after = obj.pytask_meta.after if hasattr(obj, "pytask_meta") else []
+ is_generator = obj.pytask_meta.generator if hasattr(obj, "pytask_meta") else []
# Get the underlying function to avoid having different states of the function,
# e.g. due to pytask_meta, in different layers of the wrapping.
@@ -281,7 +282,11 @@ def pytask_collect_task(
depends_on=dependencies,
produces=products,
markers=markers,
- attributes={"collection_id": collection_id, "after": after},
+ attributes={
+ "collection_id": collection_id,
+ "after": after,
+ "is_generator": is_generator,
+ },
)
return Task(
base_name=name,
@@ -290,7 +295,11 @@ def pytask_collect_task(
depends_on=dependencies,
produces=products,
markers=markers,
- attributes={"collection_id": collection_id, "after": after},
+ attributes={
+ "collection_id": collection_id,
+ "after": after,
+ "is_generator": is_generator,
+ },
)
if isinstance(obj, PTask):
return obj
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 07f83095..b7aca86c 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -7,14 +7,17 @@
from _pytask.collect_utils import collect_dependency
from _pytask.config import hookimpl
+from _pytask.exceptions import NodeLoadError
from _pytask.models import NodeInfo
from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
from _pytask.nodes import Task
+from _pytask.outcomes import CollectionOutcome
from _pytask.reports import ExecutionReport
from _pytask.tree_util import PyTree
+from _pytask.tree_util import tree_map
from _pytask.tree_util import tree_map_with_path
if TYPE_CHECKING:
@@ -34,6 +37,43 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
_recreate_dag(session, task)
+def _safe_load(node: PNode, task: PTask, is_product: bool) -> Any:
+ try:
+ return node.load(is_product=is_product)
+ except Exception as e: # noqa: BLE001
+ msg = f"Exception while loading node {node.name!r} of task {task.name!r}"
+ raise NodeLoadError(msg) from e
+
+
+@hookimpl
+def pytask_execute_task(session: Session, task: PTask) -> None:
+ """Execute task generators and collect the tasks."""
+ is_generator = task.attributes.get("is_generator", False)
+ if is_generator:
+ kwargs = {}
+ for name, value in task.depends_on.items():
+ kwargs[name] = tree_map(lambda x: _safe_load(x, task, False), value)
+
+ new_tasks = list(task.execute(**kwargs))
+
+ new_reports = []
+ for raw_task in new_tasks:
+ report = session.hook.pytask_collect_task_protocol(
+ session=session,
+ reports=session.collection_reports,
+ path=task.path if isinstance(task, PTaskWithPath) else None,
+ name=name,
+ obj=raw_task,
+ )
+ new_reports.append(report)
+
+ session.tasks.extend(
+ i.node
+ for i in new_reports
+ if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, PTask)
+ )
+
+
@hookimpl
def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
"""Check if nodes are produced by a task."""
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index f30e9f48..035b59c3 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -46,6 +46,7 @@ class CollectionMetadata:
"""
after: str | list[Callable[..., Any]] = field(factory=list)
+ generator: bool = False
id_: str | None = None
kwargs: dict[str, Any] = field(factory=dict)
markers: list[Mark] = field(factory=list)
diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py
index 947c47d1..5ab6f9f2 100644
--- a/src/_pytask/task_utils.py
+++ b/src/_pytask/task_utils.py
@@ -36,10 +36,11 @@
"""
-def task(
+def task( # noqa: PLR0913
name: str | None = None,
*,
after: str | Callable[..., Any] | list[Callable[..., Any]] | None = None,
+ generator: bool = False,
id: str | None = None, # noqa: A002
kwargs: dict[Any, Any] | None = None,
produces: PyTree[Any] | None = None,
@@ -59,6 +60,8 @@ def task(
after
An expression or a task function or a list of task functions that need to be
executed before this task can.
+ generator
+ An indicator whether this task is a task generator.
id
An id for the task if it is part of a parametrization. Otherwise, an automatic
id will be generated. See
@@ -109,20 +112,22 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
parsed_after = _parse_after(after)
if hasattr(unwrapped, "pytask_meta"):
+ unwrapped.pytask_meta.after = parsed_after
+ unwrapped.pytask_meta.generator = generator
unwrapped.pytask_meta.id_ = id
unwrapped.pytask_meta.kwargs = parsed_kwargs
unwrapped.pytask_meta.markers.append(Mark("task", (), {}))
unwrapped.pytask_meta.name = parsed_name
unwrapped.pytask_meta.produces = produces
- unwrapped.pytask_meta.after = parsed_after
else:
unwrapped.pytask_meta = CollectionMetadata(
+ after=parsed_after,
+ generator=generator,
id_=id,
kwargs=parsed_kwargs,
markers=[Mark("task", (), {})],
name=parsed_name,
produces=produces,
- after=parsed_after,
)
# Store it in the global variable ``COLLECTED_TASKS`` to avoid garbage
From e7ab6ce8a28c1d74a80b0bffdfd8a53a66232e71 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Thu, 16 Nov 2023 10:44:33 +0100
Subject: [PATCH 39/81] fix test.
---
tests/test_execute.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/tests/test_execute.py b/tests/test_execute.py
index c3c09b5d..625678e7 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1117,7 +1117,7 @@ def task_depends(
@pytest.mark.end_to_end()
-def test_gracefully_fail_when_dag_raises_error(tmp_path):
+def test_gracefully_fail_when_dag_raises_error(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DelayedPathNode, task
@@ -1137,12 +1137,12 @@ def task_depends(
"""
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
- session = build(paths=tmp_path)
- assert session.exit_code == ExitCode.OK
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
- session = build(paths=tmp_path)
- assert session.exit_code == ExitCode.FAILED
- assert session.execution_reports[1].outcome == TaskOutcome.FAIL
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.FAILED
+ assert "cycle" in result.output
@pytest.mark.end_to_end()
From 4ab7e0fcb68b3f16249cd26e2979a1665cbbe072 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Thu, 16 Nov 2023 16:04:25 +0100
Subject: [PATCH 40/81] Get preliminary version of task generators to work.
---
src/_pytask/collect.py | 4 +-
src/_pytask/dag.py | 7 ---
src/_pytask/dag_command.py | 2 +
src/_pytask/delayed.py | 100 ++++++++---------------------------
src/_pytask/delayed_utils.py | 98 ++++++++++++++++++++++++++++++++++
src/_pytask/execute.py | 13 +++--
src/_pytask/hookspecs.py | 5 --
src/_pytask/nodes.py | 1 -
src/_pytask/persist.py | 2 +
src/_pytask/skipping.py | 13 ++---
tests/test_execute.py | 9 ++--
11 files changed, 148 insertions(+), 106 deletions(-)
create mode 100644 src/_pytask/delayed_utils.py
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index b82dcfeb..66807692 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -260,7 +260,9 @@ def pytask_collect_task(
markers = get_all_marks(obj)
collection_id = obj.pytask_meta._id if hasattr(obj, "pytask_meta") else None
after = obj.pytask_meta.after if hasattr(obj, "pytask_meta") else []
- is_generator = obj.pytask_meta.generator if hasattr(obj, "pytask_meta") else []
+ is_generator = (
+ obj.pytask_meta.generator if hasattr(obj, "pytask_meta") else False
+ )
# Get the underlying function to avoid having different states of the function,
# e.g. due to pytask_meta, in different layers of the wrapping.
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index 85b480c8..1278b8aa 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -16,7 +16,6 @@
from _pytask.console import TASK_ICON
from _pytask.exceptions import ResolvingDependenciesError
from _pytask.mark import select_by_after_keyword
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
from _pytask.nodes import PythonNode
@@ -57,9 +56,6 @@ def pytask_dag_create_dag(session: Session, tasks: list[PTask]) -> nx.DiGraph:
def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
"""Add a dependency to the DAG."""
- if isinstance(node, PDelayedNode):
- return
-
dag.add_node(node.signature, node=node)
dag.add_edge(node.signature, task.signature)
@@ -71,9 +67,6 @@ def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
def _add_product(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
"""Add a product to the DAG."""
- if isinstance(node, PDelayedNode):
- return
-
dag.add_node(node.signature, node=node)
dag.add_edge(task.signature, node.signature)
diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py
index 5eeef7da..23e7b99f 100644
--- a/src/_pytask/dag_command.py
+++ b/src/_pytask/dag_command.py
@@ -247,3 +247,5 @@ def _write_graph(dag: nx.DiGraph, path: Path, layout: str) -> None:
path.parent.mkdir(exist_ok=True, parents=True)
graph = nx.nx_agraph.to_agraph(dag)
graph.draw(path, prog=layout)
+ console.print()
+ console.print(f"Written to {path}.")
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index b7aca86c..9782afa3 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -1,22 +1,20 @@
from __future__ import annotations
import sys
-from pathlib import Path
from typing import Any
from typing import TYPE_CHECKING
-from _pytask.collect_utils import collect_dependency
from _pytask.config import hookimpl
+from _pytask.delayed_utils import collect_delayed_nodes
+from _pytask.delayed_utils import recreate_dag
+from _pytask.delayed_utils import TASKS_WITH_DELAYED_NODES
from _pytask.exceptions import NodeLoadError
-from _pytask.models import NodeInfo
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
-from _pytask.nodes import Task
from _pytask.outcomes import CollectionOutcome
from _pytask.reports import ExecutionReport
-from _pytask.tree_util import PyTree
+from _pytask.task_utils import parse_collected_tasks_with_task_marker
from _pytask.tree_util import tree_map
from _pytask.tree_util import tree_map_with_path
@@ -24,17 +22,14 @@
from _pytask.session import Session
-_TASKS_WITH_DELAYED_NODES = set()
-
-
@hookimpl
def pytask_execute_task_setup(session: Session, task: PTask) -> None:
"""Collect delayed nodes and parse them."""
task.depends_on = tree_map_with_path( # type: ignore[assignment]
- lambda p, x: _collect_delayed_nodes(session, task, x, p), task.depends_on
+ lambda p, x: collect_delayed_nodes(session, task, x, p), task.depends_on
)
- if task.signature in _TASKS_WITH_DELAYED_NODES:
- _recreate_dag(session, task)
+ if task.signature in TASKS_WITH_DELAYED_NODES:
+ recreate_dag(session, task)
def _safe_load(node: PNode, task: PTask, is_product: bool) -> Any:
@@ -56,14 +51,16 @@ def pytask_execute_task(session: Session, task: PTask) -> None:
new_tasks = list(task.execute(**kwargs))
+ name_to_function = parse_collected_tasks_with_task_marker(new_tasks)
+
new_reports = []
- for raw_task in new_tasks:
+ for name, function in name_to_function.items():
report = session.hook.pytask_collect_task_protocol(
session=session,
reports=session.collection_reports,
path=task.path if isinstance(task, PTaskWithPath) else None,
name=name,
- obj=raw_task,
+ obj=function,
)
new_reports.append(report)
@@ -73,69 +70,14 @@ def pytask_execute_task(session: Session, task: PTask) -> None:
if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, PTask)
)
+ try:
+ session.hook.pytask_collect_modify_tasks(
+ session=session, tasks=session.tasks
+ )
+ except Exception: # noqa: BLE001 # pragma: no cover
+ report = ExecutionReport.from_task_and_exception(
+ task=task, exc_info=sys.exc_info()
+ )
+ session.collection_reports.append(report)
-@hookimpl
-def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
- """Check if nodes are produced by a task."""
- # Replace delayed nodes with their actually resolved nodes.
- task.produces = tree_map_with_path( # type: ignore[assignment]
- lambda p, x: _collect_delayed_nodes(session, task, x, p), task.produces
- )
-
- if task.signature in _TASKS_WITH_DELAYED_NODES:
- _recreate_dag(session, task)
-
-
-def _collect_delayed_nodes(
- session: Session, task: PTask, node: Any, path: tuple[Any, ...]
-) -> PyTree[PNode]:
- """Collect delayed nodes.
-
- 1. Call the :meth:`pytask.PDelayedNode.collect` to receive the raw nodes.
- 2. Collect the raw nodes as usual.
-
- """
- if not isinstance(node, PDelayedNode):
- return node
-
- # Add task to register to update the DAG after the task is executed.
- _TASKS_WITH_DELAYED_NODES.add(task.signature)
-
- # Collect delayed nodes and receive raw nodes.
- delayed_nodes = node.collect()
-
- # Collect raw nodes.
- node_path = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd()
- task_name = task.base_name if isinstance(task, Task) else task.name
- task_path = task.path if isinstance(task, PTaskWithPath) else None
- arg_name, *rest_path = path
-
- return tree_map_with_path(
- lambda p, x: collect_dependency(
- session,
- node_path,
- task_name,
- NodeInfo(
- arg_name=arg_name,
- path=(*rest_path, *p),
- value=x,
- task_path=task_path,
- task_name=task_name,
- ),
- ),
- delayed_nodes,
- )
-
-
-def _recreate_dag(session: Session, task: PTask) -> None:
- """Recreate the DAG."""
- try:
- session.dag = session.hook.pytask_dag_create_dag(
- session=session, tasks=session.tasks
- )
- session.hook.pytask_dag_modify_dag(session=session, dag=session.dag)
-
- except Exception: # noqa: BLE001
- report = ExecutionReport.from_task_and_exception(task, sys.exc_info())
- session.execution_reports.append(report)
- session.should_stop = True
+ recreate_dag(session, task)
diff --git a/src/_pytask/delayed_utils.py b/src/_pytask/delayed_utils.py
new file mode 100644
index 00000000..0e2f78c0
--- /dev/null
+++ b/src/_pytask/delayed_utils.py
@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+from typing import TYPE_CHECKING
+
+from _pytask.collect_utils import collect_dependency
+from _pytask.dag_utils import TopologicalSorter
+from _pytask.models import NodeInfo
+from _pytask.node_protocols import PDelayedNode
+from _pytask.node_protocols import PNode
+from _pytask.node_protocols import PTask
+from _pytask.node_protocols import PTaskWithPath
+from _pytask.nodes import Task
+from _pytask.reports import ExecutionReport
+from _pytask.tree_util import PyTree
+from _pytask.tree_util import tree_map_with_path
+
+if TYPE_CHECKING:
+ from _pytask.session import Session
+
+
+TASKS_WITH_DELAYED_NODES = set()
+
+
+def collect_delayed_nodes(
+ session: Session, task: PTask, node: Any, path: tuple[Any, ...]
+) -> PyTree[PNode]:
+ """Collect delayed nodes.
+
+ 1. Call the :meth:`pytask.PDelayedNode.collect` to receive the raw nodes.
+ 2. Collect the raw nodes as usual.
+
+ """
+ if not isinstance(node, PDelayedNode):
+ return node
+
+ # Add task to register to update the DAG after the task is executed.
+ TASKS_WITH_DELAYED_NODES.add(task.signature)
+
+ # Collect delayed nodes and receive raw nodes.
+ delayed_nodes = node.collect()
+
+ # Collect raw nodes.
+ node_path = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd()
+ task_name = task.base_name if isinstance(task, Task) else task.name
+ task_path = task.path if isinstance(task, PTaskWithPath) else None
+ arg_name, *rest_path = path
+
+ return tree_map_with_path(
+ lambda p, x: collect_dependency(
+ session,
+ node_path,
+ task_name,
+ NodeInfo(
+ arg_name=arg_name,
+ path=(*rest_path, *p),
+ value=x,
+ task_path=task_path,
+ task_name=task_name,
+ ),
+ ),
+ delayed_nodes,
+ )
+
+
+def recreate_dag(session: Session, task: PTask) -> None:
+ """Recreate the DAG."""
+ try:
+ session.dag = session.hook.pytask_dag_create_dag(
+ session=session, tasks=session.tasks
+ )
+ session.hook.pytask_dag_modify_dag(session=session, dag=session.dag)
+ session.scheduler = TopologicalSorter.from_dag_and_sorter(
+ session.dag, session.scheduler
+ )
+
+ except Exception: # noqa: BLE001
+ report = ExecutionReport.from_task_and_exception(task, sys.exc_info())
+ session.execution_reports.append(report)
+ session.should_stop = True
+
+
+def collect_delayed_products(session: Session, task: PTask) -> None:
+ """Collect delayed products.
+
+ Unfortunately, this function needs to be called when a task finishes successfully
+ (skipped unchanged, persisted, etc..).
+
+ """
+ # Replace delayed nodes with their actually resolved nodes.
+ task.produces = tree_map_with_path( # type: ignore[assignment]
+ lambda p, x: collect_delayed_nodes(session, task, x, p), task.produces
+ )
+
+ if task.signature in TASKS_WITH_DELAYED_NODES:
+ recreate_dag(session, task)
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 016fe5dc..1d3ce2cb 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -20,6 +20,7 @@
from _pytask.dag_utils import TopologicalSorter
from _pytask.database_utils import has_node_changed
from _pytask.database_utils import update_states_in_database
+from _pytask.delayed_utils import collect_delayed_products
from _pytask.exceptions import ExecutionError
from _pytask.exceptions import NodeLoadError
from _pytask.exceptions import NodeNotFoundError
@@ -92,8 +93,6 @@ def pytask_execute_build(session: Session) -> bool | None:
session.execution_reports.append(report)
session.scheduler.done(task_name)
- session.hook.pytask_execute_collect_delayed_tasks(session=session)
-
if session.should_stop:
return True
return True
@@ -123,7 +122,7 @@ def pytask_execute_task_protocol(session: Session, task: PTask) -> ExecutionRepo
@hookimpl(trylast=True)
-def pytask_execute_task_setup(session: Session, task: PTask) -> None:
+def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C901
"""Set up the execution of a task.
1. Check whether all dependencies of a task are available.
@@ -151,12 +150,17 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
)
raise NodeNotFoundError(msg)
+ # Skip delayed nodes that are products since they do not have a state.
+ if node_signature not in predecessors and isinstance(node, PDelayedNode):
+ continue
+
has_changed = has_node_changed(task=task, node=node)
if has_changed:
needs_to_be_executed = True
break
if not needs_to_be_executed:
+ collect_delayed_products(session, task)
raise SkippedUnchanged
# Create directory for product if it does not exist. Maybe this should be a `setup`
@@ -218,6 +222,7 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
@hookimpl(trylast=True)
def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
"""Check if nodes are produced by a task."""
+ collect_delayed_products(session, task)
missing_nodes = [node for node in tree_leaves(task.produces) if not node.state()]
if missing_nodes:
paths = session.config["paths"]
@@ -228,7 +233,7 @@ def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
raise NodeNotFoundError(formatted)
-@hookimpl(trylast=True)
+@hookimpl
def pytask_execute_task_process_report(
session: Session, report: ExecutionReport
) -> bool:
diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py
index 5680feff..8e944e9f 100644
--- a/src/_pytask/hookspecs.py
+++ b/src/_pytask/hookspecs.py
@@ -357,11 +357,6 @@ def pytask_execute_task_log_end(session: Session, report: ExecutionReport) -> No
"""Log the end of a task execution."""
-@hookspec
-def pytask_execute_collect_delayed_tasks(session: Session) -> list[PTask]:
- """Collect delayed tasks."""
-
-
@hookspec
def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> None:
"""Log the footer of the execution report."""
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 94d51ae7..225d75bf 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -361,7 +361,6 @@ class DelayedPathNode(PNode):
@property
def signature(self) -> str:
"""The unique signature of the node."""
- raise NotImplementedError
raw_key = "".join(str(hash_value(arg)) for arg in (self.root_dir, self.pattern))
return hashlib.sha256(raw_key.encode()).hexdigest()
diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py
index 49b61510..54de6b28 100644
--- a/src/_pytask/persist.py
+++ b/src/_pytask/persist.py
@@ -8,6 +8,7 @@
from _pytask.dag_utils import node_and_neighbors
from _pytask.database_utils import has_node_changed
from _pytask.database_utils import update_states_in_database
+from _pytask.delayed_utils import collect_delayed_products
from _pytask.mark_utils import has_mark
from _pytask.outcomes import Persisted
from _pytask.outcomes import TaskOutcome
@@ -56,6 +57,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
for name in node_and_neighbors(session.dag, task.signature)
)
if any_node_changed:
+ collect_delayed_products(session, task)
raise Persisted
diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py
index e86d3d72..18b50554 100644
--- a/src/_pytask/skipping.py
+++ b/src/_pytask/skipping.py
@@ -6,6 +6,7 @@
from _pytask.config import hookimpl
from _pytask.dag_utils import descending_tasks
+from _pytask.delayed_utils import collect_delayed_products
from _pytask.mark import Mark
from _pytask.mark_utils import get_marks
from _pytask.mark_utils import has_mark
@@ -52,6 +53,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
task, "would_be_executed"
)
if is_unchanged and not session.config["force"]:
+ collect_delayed_products(session, task)
raise SkippedUnchanged
is_skipped = has_mark(task, "skip")
@@ -89,8 +91,9 @@ def pytask_execute_task_process_report(
if report.exc_info:
if isinstance(report.exc_info[1], SkippedUnchanged):
report.outcome = TaskOutcome.SKIP_UNCHANGED
+ return True
- elif isinstance(report.exc_info[1], Skipped):
+ if isinstance(report.exc_info[1], Skipped):
report.outcome = TaskOutcome.SKIP
for descending_task_name in descending_tasks(task.signature, session.dag):
@@ -102,12 +105,10 @@ def pytask_execute_task_process_report(
{"reason": f"Previous task {task.name!r} was skipped."},
)
)
+ return True
- elif isinstance(report.exc_info[1], SkippedAncestorFailed):
+ if isinstance(report.exc_info[1], SkippedAncestorFailed):
report.outcome = TaskOutcome.SKIP_PREVIOUS_FAILED
+ return True
- if report.exc_info and isinstance(
- report.exc_info[1], (Skipped, SkippedUnchanged, SkippedAncestorFailed)
- ):
- return True
return None
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 625678e7..530f2033 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1142,7 +1142,7 @@ def task_depends(
result = runner.invoke(cli, [tmp_path.as_posix()])
assert result.exit_code == ExitCode.FAILED
- assert "cycle" in result.output
+ assert "There are some tasks which produce" in result.output
@pytest.mark.end_to_end()
@@ -1157,12 +1157,13 @@ def task_produces() -> Annotated[None, DelayedPathNode(pattern="[ab].txt")]:
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
- @task(after=task_produces)
+ @task(after=task_produces, generator=True)
def task_depends(
paths = DelayedPathNode(pattern="[ab].txt")
) -> ...:
for path in paths:
+ @task
def task_copy(
path: Path = path
) -> Annotated[str, path.with_name(path.stem + "-copy.txt")]:
@@ -1175,6 +1176,8 @@ def task_copy(
session = build(paths=tmp_path)
assert session.exit_code == ExitCode.OK
- assert len(session.tasks) == 2
+ assert len(session.tasks) == 4
assert len(session.tasks[0].produces["return"]) == 2
assert len(session.tasks[1].depends_on["paths"]) == 2
+ assert tmp_path.joinpath("a-copy.txt").exists()
+ assert tmp_path.joinpath("b-copy.txt").exists()
From cabb5314b70f17caf2df396d718647d1c6bfccda Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Thu, 16 Nov 2023 17:12:37 +0100
Subject: [PATCH 41/81] Temp.
---
src/_pytask/delayed.py | 40 +++++++++++++--
src/_pytask/task.py | 5 ++
src/_pytask/traceback.py | 6 +--
tests/test_execute.py | 105 +++++++++++++++++++++++++++++++++++++++
4 files changed, 150 insertions(+), 6 deletions(-)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 9782afa3..cfaa680a 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -1,7 +1,10 @@
from __future__ import annotations
+import inspect
import sys
from typing import Any
+from typing import Callable
+from typing import Mapping
from typing import TYPE_CHECKING
from _pytask.config import hookimpl
@@ -14,9 +17,11 @@
from _pytask.node_protocols import PTaskWithPath
from _pytask.outcomes import CollectionOutcome
from _pytask.reports import ExecutionReport
+from _pytask.task_utils import COLLECTED_TASKS
from _pytask.task_utils import parse_collected_tasks_with_task_marker
from _pytask.tree_util import tree_map
from _pytask.tree_util import tree_map_with_path
+from _pytask.typing import is_task_function
if TYPE_CHECKING:
from _pytask.session import Session
@@ -40,8 +45,13 @@ def _safe_load(node: PNode, task: PTask, is_product: bool) -> Any:
raise NodeLoadError(msg) from e
+_ERROR_TASK_GENERATOR_RETURN = """\
+Could not collect return of task generator. The return should be a task function or a \
+task class but received {obj} instead."""
+
+
@hookimpl
-def pytask_execute_task(session: Session, task: PTask) -> None:
+def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, PLR0912
"""Execute task generators and collect the tasks."""
is_generator = task.attributes.get("is_generator", False)
if is_generator:
@@ -49,9 +59,33 @@ def pytask_execute_task(session: Session, task: PTask) -> None:
for name, value in task.depends_on.items():
kwargs[name] = tree_map(lambda x: _safe_load(x, task, False), value)
- new_tasks = list(task.execute(**kwargs))
+ out = task.execute(**kwargs)
+ if inspect.isgenerator(out):
+ out = list(out)
- name_to_function = parse_collected_tasks_with_task_marker(new_tasks)
+ # Parse tasks created with @task.
+ name_to_function: Mapping[str, Callable[..., Any] | PTask]
+ if isinstance(task, PTaskWithPath) and task.path in COLLECTED_TASKS:
+ tasks = COLLECTED_TASKS.pop(task.path)
+ name_to_function = parse_collected_tasks_with_task_marker(tasks)
+ else:
+ # Parse individual tasks.
+ if is_task_function(out) or isinstance(out, PTask):
+ out = [out]
+ # Parse tasks from iterable.
+ if hasattr(out, "__iter__"):
+ name_to_function = {}
+ for obj in out:
+ if is_task_function(obj):
+ name_to_function[obj.__name__] = obj
+ elif isinstance(obj, PTask):
+ name_to_function[obj.name] = obj
+ else:
+ msg = _ERROR_TASK_GENERATOR_RETURN.format(obj)
+ raise ValueError(msg)
+ else:
+ msg = _ERROR_TASK_GENERATOR_RETURN.format(out)
+ raise ValueError(msg)
new_reports = []
for name, function in name_to_function.items():
diff --git a/src/_pytask/task.py b/src/_pytask/task.py
index 7f898721..caa95b36 100644
--- a/src/_pytask/task.py
+++ b/src/_pytask/task.py
@@ -79,3 +79,8 @@ def _raise_error_when_task_functions_are_duplicated(
f"the '@task(...)' decorator.\n\n{flat_tree}"
)
raise ValueError(msg)
+
+
+@hookimpl
+def pytask_unconfigure() -> None:
+ COLLECTED_TASKS.clear()
diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py
index c3f0ea61..d476f39d 100644
--- a/src/_pytask/traceback.py
+++ b/src/_pytask/traceback.py
@@ -47,9 +47,9 @@ class Traceback:
show_locals: ClassVar[bool] = False
suppress: ClassVar[tuple[Path, ...]] = (
- _PLUGGY_DIRECTORY,
- TREE_UTIL_LIB_DIRECTORY,
- _PYTASK_DIRECTORY,
+ # _PLUGGY_DIRECTORY,
+ # TREE_UTIL_LIB_DIRECTORY,
+ # _PYTASK_DIRECTORY,
)
def __rich_console__(
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 530f2033..d27afaff 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1152,6 +1152,42 @@ def test_delayed_task_generation(tmp_path):
from pytask import DelayedPathNode, task
from pathlib import Path
+ def task_produces() -> Annotated[None, DelayedPathNode(pattern="[ab].txt")]:
+ path = Path(__file__).parent
+ path.joinpath("a.txt").write_text("Hello, ")
+ path.joinpath("b.txt").write_text("World!")
+
+ @task(after=task_produces, generator=True)
+ def task_depends(
+ paths = DelayedPathNode(pattern="[ab].txt")
+ ) -> ...:
+ for path in paths:
+
+ @task
+ def task_copy(
+ path: Path = path
+ ) -> Annotated[str, path.with_name(path.stem + "-copy.txt")]:
+ return path.read_text()
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ session = build(paths=tmp_path)
+
+ assert session.exit_code == ExitCode.OK
+ assert len(session.tasks) == 4
+ assert len(session.tasks[0].produces["return"]) == 2
+ assert len(session.tasks[1].depends_on["paths"]) == 2
+ assert tmp_path.joinpath("a-copy.txt").exists()
+ assert tmp_path.joinpath("b-copy.txt").exists()
+
+
+@pytest.mark.end_to_end()
+def test_delayed_task_generation_with_generator(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task
+ from pathlib import Path
+
def task_produces() -> Annotated[None, DelayedPathNode(pattern="[ab].txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
@@ -1181,3 +1217,72 @@ def task_copy(
assert len(session.tasks[1].depends_on["paths"]) == 2
assert tmp_path.joinpath("a-copy.txt").exists()
assert tmp_path.joinpath("b-copy.txt").exists()
+
+
+@pytest.mark.end_to_end()
+def test_delayed_task_generation_with_single_function(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task
+ from pathlib import Path
+
+ def task_produces() -> Annotated[None, DelayedPathNode(pattern="[a].txt")]:
+ path = Path(__file__).parent
+ path.joinpath("a.txt").write_text("Hello, ")
+
+ @task(after=task_produces, generator=True)
+ def task_depends(
+ paths = DelayedPathNode(pattern="[a].txt")
+ ) -> ...:
+ path = paths[0]
+
+ def task_copy(
+ path: Path = path
+ ) -> Annotated[str, path.with_name(path.stem + "-copy.txt")]:
+ return path.read_text()
+ return task_copy
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ session = build(paths=tmp_path)
+
+ assert session.exit_code == ExitCode.OK
+ assert len(session.tasks) == 3
+ assert len(session.tasks[0].produces["return"]) == 1
+ assert len(session.tasks[1].depends_on["paths"]) == 1
+ assert tmp_path.joinpath("a-copy.txt").exists()
+
+
+@pytest.mark.end_to_end()
+def test_delayed_task_generation_with_task_node(tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, TaskWithoutPath, task, PathNode
+ from pathlib import Path
+
+ def task_produces() -> Annotated[None, DelayedPathNode(pattern="[a].txt")]:
+ path = Path(__file__).parent
+ path.joinpath("a.txt").write_text("Hello, ")
+
+ @task(after=task_produces, generator=True)
+ def task_depends(
+ paths = DelayedPathNode(pattern="[a].txt")
+ ) -> ...:
+ path = paths[0]
+
+ task_copy = TaskWithoutPath(
+ name="task_copy",
+ function=lambda path: path.read_text(),
+ produces={"return": PathNode(path=path.with_name(path.stem + "-copy.txt"))},
+ )
+ return task_copy
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ session = build(paths=tmp_path)
+
+ assert session.exit_code == ExitCode.OK
+ assert len(session.tasks) == 3
+ assert len(session.tasks[0].produces["return"]) == 1
+ assert len(session.tasks[1].depends_on["paths"]) == 1
+ assert tmp_path.joinpath("a-copy.txt").exists()
From 3ab8bac6a4e57556f138d369f53c4bf676f6a543 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Thu, 16 Nov 2023 17:12:57 +0100
Subject: [PATCH 42/81] Temp.
---
src/_pytask/traceback.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/_pytask/traceback.py b/src/_pytask/traceback.py
index d476f39d..c3f0ea61 100644
--- a/src/_pytask/traceback.py
+++ b/src/_pytask/traceback.py
@@ -47,9 +47,9 @@ class Traceback:
show_locals: ClassVar[bool] = False
suppress: ClassVar[tuple[Path, ...]] = (
- # _PLUGGY_DIRECTORY,
- # TREE_UTIL_LIB_DIRECTORY,
- # _PYTASK_DIRECTORY,
+ _PLUGGY_DIRECTORY,
+ TREE_UTIL_LIB_DIRECTORY,
+ _PYTASK_DIRECTORY,
)
def __rich_console__(
From 7644b3565c0a5dc23fbbf2d45ea385c6dead5902 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Thu, 16 Nov 2023 17:56:32 +0100
Subject: [PATCH 43/81] Fix.
---
tests/test_execute.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/tests/test_execute.py b/tests/test_execute.py
index dd3a4b98..5935fac0 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1313,6 +1313,7 @@ def task_depends(
task_copy = TaskWithoutPath(
name="task_copy",
function=lambda path: path.read_text(),
+ depends_on={"path": PathNode(path=path)},
produces={"return": PathNode(path=path.with_name(path.stem + "-copy.txt"))},
)
return task_copy
From 2bff2863f26a38d8bea10a68cbf1861d7850288d Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 17 Nov 2023 09:40:58 +0100
Subject: [PATCH 44/81] Fix.
---
src/_pytask/collect.py | 9 +--------
src/_pytask/data_catalog.py | 6 +-----
src/_pytask/delayed.py | 5 +++++
src/_pytask/execute.py | 2 +-
tests/test_task_utils.py | 2 ++
5 files changed, 10 insertions(+), 14 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index eb9a0822..0c7b026f 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -510,14 +510,7 @@ def pytask_collect_log(
"""Log collection."""
session.collection_end = time.time()
- msg = f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}."
- if session.delayed_tasks:
- n_delayed_tasks = len(session.delayed_tasks)
- msg = (
- msg[:-1] + f" and {n_delayed_tasks} delayed "
- f"task{'' if n_delayed_tasks == 1 else 's'}."
- )
- console.print(msg)
+ console.print(f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.")
failed_reports = [r for r in reports if r.outcome == CollectionOutcome.FAIL]
if failed_reports:
diff --git a/src/_pytask/data_catalog.py b/src/_pytask/data_catalog.py
index ce8eb064..84c25375 100644
--- a/src/_pytask/data_catalog.py
+++ b/src/_pytask/data_catalog.py
@@ -119,11 +119,7 @@ def add(self, name: str, node: DataCatalog | PNode | None = None) -> None:
session=self._session,
path=self._instance_path,
node_info=NodeInfo(
- arg_name=name,
- path=(),
- value=node,
- task_path=None,
- task_name="",
+ arg_name=name, path=(), value=node, task_path=None, task_name=""
),
)
if collected_node is None: # pragma: no cover
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index cfaa680a..3d625957 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -115,3 +115,8 @@ def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, P
session.collection_reports.append(report)
recreate_dag(session, task)
+
+
+@hookimpl
+def pytask_unconfigure() -> None:
+ TASKS_WITH_DELAYED_NODES.clear()
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 1d3ce2cb..166abea2 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -233,7 +233,7 @@ def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
raise NodeNotFoundError(formatted)
-@hookimpl
+@hookimpl(trylast=True)
def pytask_execute_task_process_report(
session: Session, report: ExecutionReport
) -> bool:
diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py
index aa5cde7f..632d410f 100644
--- a/tests/test_task_utils.py
+++ b/tests/test_task_utils.py
@@ -66,6 +66,8 @@ def test_default_values_of_pytask_meta():
def task_example():
...
+ assert task_example.pytask_meta.after == []
+ assert not task_example.pytask_meta.generator
assert task_example.pytask_meta.id_ is None
assert task_example.pytask_meta.kwargs == {}
assert task_example.pytask_meta.markers == [Mark("task", (), {})]
From c1c1e7a92af13294d31a8150c6d1dac3b3984c4d Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 17 Nov 2023 11:01:12 +0100
Subject: [PATCH 45/81] Refine task generators even more.
---
src/_pytask/delayed.py | 25 +++++++++++++++--
src/_pytask/delayed_utils.py | 4 +++
src/_pytask/execute.py | 9 +++++-
src/_pytask/node_protocols.py | 37 ++++++++++++++++++++----
src/_pytask/profile.py | 2 +-
src/_pytask/typing.py | 6 ++++
tests/test_execute.py | 53 +++++++++++++++++++++++++++++++++++
7 files changed, 127 insertions(+), 9 deletions(-)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 3d625957..062d8d30 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -22,6 +22,8 @@
from _pytask.tree_util import tree_map
from _pytask.tree_util import tree_map_with_path
from _pytask.typing import is_task_function
+from _pytask.typing import is_task_generator
+from pytask import TaskOutcome
if TYPE_CHECKING:
from _pytask.session import Session
@@ -53,12 +55,16 @@ def _safe_load(node: PNode, task: PTask, is_product: bool) -> Any:
@hookimpl
def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, PLR0912
"""Execute task generators and collect the tasks."""
- is_generator = task.attributes.get("is_generator", False)
- if is_generator:
+ if is_task_generator(task):
kwargs = {}
for name, value in task.depends_on.items():
kwargs[name] = tree_map(lambda x: _safe_load(x, task, False), value)
+ parameters = inspect.signature(task.function).parameters
+ for name, value in task.produces.items():
+ if name in parameters:
+ kwargs[name] = tree_map(lambda x: _safe_load(x, task, True), value)
+
out = task.execute(**kwargs)
if inspect.isgenerator(out):
out = list(out)
@@ -117,6 +123,21 @@ def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, P
recreate_dag(session, task)
+@hookimpl
+def pytask_execute_task_process_report(report: ExecutionReport) -> bool | None:
+ """Prevent update of states for successful task generators.
+
+ It also leads to task generators always being executed, but we have an additional
+ switch implemented in ``pytask_execute_task_setup``.
+
+ """
+ task = report.task
+ if report.outcome == TaskOutcome.SUCCESS and is_task_generator(task):
+ return True
+ return None
+
+
@hookimpl
def pytask_unconfigure() -> None:
+ """Clear the global variable after execution."""
TASKS_WITH_DELAYED_NODES.clear()
diff --git a/src/_pytask/delayed_utils.py b/src/_pytask/delayed_utils.py
index 0e2f78c0..ffcbfc55 100644
--- a/src/_pytask/delayed_utils.py
+++ b/src/_pytask/delayed_utils.py
@@ -16,6 +16,7 @@
from _pytask.reports import ExecutionReport
from _pytask.tree_util import PyTree
from _pytask.tree_util import tree_map_with_path
+from _pytask.typing import is_task_generator
if TYPE_CHECKING:
from _pytask.session import Session
@@ -89,6 +90,9 @@ def collect_delayed_products(session: Session, task: PTask) -> None:
(skipped unchanged, persisted, etc..).
"""
+ if is_task_generator(task):
+ return
+
# Replace delayed nodes with their actually resolved nodes.
task.produces = tree_map_with_path( # type: ignore[assignment]
lambda p, x: collect_delayed_nodes(session, task, x, p), task.produces
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 166abea2..fa621228 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -40,6 +40,7 @@
from _pytask.tree_util import tree_leaves
from _pytask.tree_util import tree_map
from _pytask.tree_util import tree_structure
+from _pytask.typing import is_task_generator
from rich.text import Text
@@ -134,7 +135,10 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C
dag = session.dag
- needs_to_be_executed = session.config["force"]
+ # Task generators are always executed since their states are not updated, but we
+ # skip the checks as well.
+ needs_to_be_executed = session.config["force"] or is_task_generator(task)
+
if not needs_to_be_executed:
predecessors = set(dag.predecessors(task.signature)) | {task.signature}
for node_signature in node_and_neighbors(dag, task.signature):
@@ -222,6 +226,9 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
@hookimpl(trylast=True)
def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
"""Check if nodes are produced by a task."""
+ if is_task_generator(task):
+ return
+
collect_delayed_products(session, task)
missing_nodes = [node for node in tree_leaves(task.produces) if not node.state()]
if missing_nodes:
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index bf0903e2..5b2e8e8b 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -101,14 +101,41 @@ class PTaskWithPath(PTask, Protocol):
@runtime_checkable
-class PDelayedNode(Protocol):
+class PDelayedNode(PNode, Protocol):
"""A protocol for delayed nodes.
- Delayed nodes are nodes that define how nodes look like instead of the actual nodes.
- Situations like this can happen if tasks produce an unknown amount of nodes, but the
- style is known.
+ Delayed nodes are called delayed because they are resolved to actual nodes,
+ :class:`PNode`, right before a task is executed as a dependency and after the task
+ is executed as a product.
+
+ Delayed nodes are nodes that define how the actual nodes look like. They can be
+ useful when, for example, a task produces an unknown amount of nodes because it
+ downloads some files.
"""
+ def load(self, is_product: bool = False) -> Any:
+ """The load method of a delayed node.
+
+ A delayed node will never be loaded as a dependency since it would be collected
+ before.
+
+ It is possible to load a delayed node as a dependency so that it can inject
+ basic information about it in the task. For example,
+ :meth:`pytask.DelayedPathNode` injects the root directory.
+
+ """
+ if is_product:
+ ...
+ raise NotImplementedError
+
+ def save(self, value: Any) -> None:
+ """A delayed node can never save a value."""
+ raise NotImplementedError
+
+ def state(self) -> None:
+ """A delayed node has not state."""
+ raise NotImplementedError
+
def collect(self) -> list[Any]:
- """Collect the objects that are defined by the fuzzy node."""
+ """Collect the objects that are defined by the delayed nodes."""
diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py
index 4c873f5d..c9b0ac5d 100644
--- a/src/_pytask/profile.py
+++ b/src/_pytask/profile.py
@@ -46,7 +46,7 @@ class _ExportFormats(enum.Enum):
CSV = "csv"
-class Runtime(BaseTable):
+class Runtime(BaseTable): # type: ignore[valid-type, misc]
"""Record of runtimes of tasks."""
__tablename__ = "runtime"
diff --git a/src/_pytask/typing.py b/src/_pytask/typing.py
index 22165c75..ef46ddc0 100644
--- a/src/_pytask/typing.py
+++ b/src/_pytask/typing.py
@@ -10,6 +10,7 @@
from attrs import define
if TYPE_CHECKING:
+ from pytask import PTask
from typing_extensions import TypeAlias
@@ -32,6 +33,11 @@ def is_task_function(obj: Any) -> bool:
)
+def is_task_generator(task: PTask) -> bool:
+ """Check if a task is a generator."""
+ return task.attributes.get("is_generator", False)
+
+
class _NoDefault(Enum):
"""A singleton for no defaults.
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 5935fac0..2ed85b85 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1327,3 +1327,56 @@ def task_depends(
assert len(session.tasks[0].produces["return"]) == 1
assert len(session.tasks[1].depends_on["paths"]) == 1
assert tmp_path.joinpath("a-copy.txt").exists()
+
+
+@pytest.mark.end_to_end()
+def test_gracefully_fail_when_task_generator_raises_error(runner, tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task, Product
+ from pathlib import Path
+
+ @task(generator=True)
+ def task_example(
+ root_dir: Annotated[Path, DelayedPathNode(pattern="[a].txt"), Product]
+ ) -> ...:
+ raise Exception
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.FAILED
+ assert "1 Collected task" in result.output
+ assert "1 Failed" in result.output
+
+
+@pytest.mark.end_to_end()
+def test_use_delayed_node_as_product_in_generator_without_rerun(runner, tmp_path):
+ source = """
+ from typing_extensions import Annotated
+ from pytask import DelayedPathNode, task, Product
+ from pathlib import Path
+
+ @task(generator=True)
+ def task_example(
+ root_dir: Annotated[Path, DelayedPathNode(pattern="[ab].txt"), Product]
+ ) -> ...:
+ for path in (root_dir / "a.txt", root_dir / "b.txt"):
+
+ @task
+ def create_file() -> Annotated[Path, path]:
+ return "content"
+ """
+ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "3 Collected task" in result.output
+ assert "3 Succeeded" in result.output
+
+ # No rerun.
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "3 Collected task" in result.output
+ assert "1 Succeeded" in result.output
+ assert "2 Skipped because unchanged" in result.output
From 421db8eb36bcec5b5c2ea7c96ce426091fd0ad8c Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 18 Nov 2023 08:16:12 +0100
Subject: [PATCH 46/81] fix some.
---
docs/source/reference_guides/api.md | 4 ++++
src/_pytask/models.py | 8 +-------
src/_pytask/nodes.py | 10 +++-------
src/_pytask/session.py | 4 ----
src/_pytask/task_utils.py | 2 +-
5 files changed, 9 insertions(+), 19 deletions(-)
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index 42ca4d1f..15496420 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -255,6 +255,8 @@ Protocols define how tasks and nodes for dependencies and products have to be se
:show-inheritance:
.. autoprotocol:: pytask.PTaskWithPath
:show-inheritance:
+.. autoprotocol:: pytask.PDelayedNode
+ :show-inheritance:
```
## Nodes
@@ -268,6 +270,8 @@ Nodes are the interface for different kinds of dependencies or products.
:members: load, save
.. autoclass:: pytask.PythonNode
:members: load, save
+.. autoclass:: pytask.DelayedPathNode
+ :members: load, collect
```
To parse dependencies and products from nodes, use the following functions.
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index 035b59c3..2babee1c 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -46,7 +46,7 @@ class CollectionMetadata:
"""
after: str | list[Callable[..., Any]] = field(factory=list)
- generator: bool = False
+ is_generator: bool = False
id_: str | None = None
kwargs: dict[str, Any] = field(factory=dict)
markers: list[Mark] = field(factory=list)
@@ -61,9 +61,3 @@ class NodeInfo(NamedTuple):
task_path: Path | None
task_name: str
value: Any
-
-
-class DelayedTask(NamedTuple):
- path: Path | None
- name: str
- obj: Any
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 225d75bf..c6358c02 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -10,6 +10,7 @@
from typing import TYPE_CHECKING
from _pytask._hashlib import hash_value
+from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
@@ -333,8 +334,8 @@ def save(self, value: Any) -> None:
@define(kw_only=True)
-class DelayedPathNode(PNode):
- """A class for delayed :class:`PathNode`s.
+class DelayedPathNode(PDelayedNode):
+ """A class for delayed :class:`PathNode`.
Attributes
----------
@@ -347,11 +348,6 @@ class DelayedPathNode(PNode):
name
The name of the node.
- .. warning::
-
- The node inherits from PNode although it does not provide a state, load, and
- save method.
-
"""
root_dir: Path | None = None
diff --git a/src/_pytask/session.py b/src/_pytask/session.py
index f3617628..a28aa2d8 100644
--- a/src/_pytask/session.py
+++ b/src/_pytask/session.py
@@ -16,7 +16,6 @@
from pluggy._hooks import _HookRelay as HookRelay
if TYPE_CHECKING:
- from _pytask.models import DelayedTask
from _pytask.node_protocols import PTask
from _pytask.warnings_utils import WarningReport
from _pytask.reports import CollectionReport
@@ -36,8 +35,6 @@ class Session:
Reports for collected items.
dag
The DAG of the project.
- delayed_tasks
- List of all delayed tasks that are collected once they are ready.
hook
Holds all hooks collected by pytask.
tasks
@@ -58,7 +55,6 @@ class Session:
config: dict[str, Any] = field(factory=dict)
collection_reports: list[CollectionReport] = field(factory=list)
dag: nx.DiGraph = field(factory=nx.DiGraph)
- delayed_tasks: list[DelayedTask] = field(factory=list)
hook: HookRelay = field(factory=HookRelay)
tasks: list[PTask] = field(factory=list)
dag_report: DagReport | None = None
diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py
index 1e2e5808..297f8505 100644
--- a/src/_pytask/task_utils.py
+++ b/src/_pytask/task_utils.py
@@ -123,7 +123,7 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
else:
unwrapped.pytask_meta = CollectionMetadata(
after=parsed_after,
- generator=generator,
+ is_generator=generator,
id_=id,
kwargs=parsed_kwargs,
markers=[Mark("task", (), {})],
From 1d9e85bfc55c20458b90743c7d15e065135a43c4 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 22 Nov 2023 23:31:51 +0100
Subject: [PATCH 47/81] Add docs.
---
docs/source/how_to_guides/delayed_tasks.md | 76 +++++++++++++++++++
docs/source/how_to_guides/index.md | 1 +
.../defining_dependencies_products.md | 2 +
.../delayed_tasks_delayed_products.py | 25 ++++++
.../delayed_tasks_delayed_task.py | 14 ++++
.../delayed_tasks_task_generator.py | 21 +++++
src/_pytask/collect.py | 2 +-
src/_pytask/models.py | 2 +
src/_pytask/task_utils.py | 2 +-
tests/test_task_utils.py | 2 +-
10 files changed, 144 insertions(+), 3 deletions(-)
create mode 100644 docs/source/how_to_guides/delayed_tasks.md
create mode 100644 docs_src/how_to_guides/delayed_tasks_delayed_products.py
create mode 100644 docs_src/how_to_guides/delayed_tasks_delayed_task.py
create mode 100644 docs_src/how_to_guides/delayed_tasks_task_generator.py
diff --git a/docs/source/how_to_guides/delayed_tasks.md b/docs/source/how_to_guides/delayed_tasks.md
new file mode 100644
index 00000000..4783220f
--- /dev/null
+++ b/docs/source/how_to_guides/delayed_tasks.md
@@ -0,0 +1,76 @@
+# Delayed tasks
+
+pytask's execution model is can usually be viewed as three phases.
+
+1. Collection of tasks, dependencies, and products.
+1. Building the DAG.
+1. Executing the tasks.
+
+But, in some situations pytask needs to be more flexible.
+
+Imagine you want to download files from some online storage, but the total number of
+files and their filenames is unknown before the task has started. How can you describe
+the files still as products of the task?
+
+And how would you define a task that depends on these files. Or, how would define a
+single task to process each file.
+
+The following sections will explain how delayed tasks work with pytask.
+
+## Delayed Products
+
+Let us start with a task that downloads an unknown amount of files and stores them on
+disk in a folder called `downloads`. As an example, we will download all files without a
+file extension from the pytask repository.
+
+```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_delayed_products.py
+---
+emphasize-lines: 4, 11
+---
+```
+
+Since the names of the filesare not known when pytask is started, we need to use a
+{class}`~pytask.DelayedPathNode`. With a {class}`~pytask.DelayedPathNode` we can specify
+where pytask can find the files and how they look like with an optional path and a glob
+pattern.
+
+When we use the {class}`~pytask.DelayedPathNode` as a product annotation, we get access
+to the `root_dir` as a {class}`~pathlib.Path` object inside the function which allows us
+to store the files.
+
+## Delayed task
+
+In the next step, we want to define a task that consumes all previously downloaded files
+and merges them into one file.
+
+```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_delayed_task.py
+---
+emphasize-lines: 8-10
+---
+```
+
+When {class}`~pytask.DelayedPathNode` is used as a dependency a list of all the files in
+the folder defined by the root path and the pattern are automatically collected and
+passed to the task.
+
+As long as we use a {class}`DelayedPathNode` with the same `root_dir` and `pattern` in
+both tasks, pytask will automatically recognize that the second task depends on the
+first. If that is not true, you might need to make this dependency more explicit by
+using {func}`@task(after=...) <pytask.task>` which is explained {ref}`here <after>`.
+
+## Delayed and repeated tasks
+
+Coming to the last use-case, what if we wanted to process each of the downloaded files
+separately instead of dealing with them in one task?
+
+To define an unknown amount of tasks for an unknown amount of downloaded files, we have
+to write a task generator.
+
+A task generator is a task function in which we define more tasks, just as if we were
+writing functions in a normal task module.
+
+As an example, each task takes one of the downloaded files and copies its content to a
+`.txt` file.
+
+```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_task_generator.py
+```
diff --git a/docs/source/how_to_guides/index.md b/docs/source/how_to_guides/index.md
index 5b5fb660..eb062e2d 100644
--- a/docs/source/how_to_guides/index.md
+++ b/docs/source/how_to_guides/index.md
@@ -17,6 +17,7 @@ capture_warnings
how_to_influence_build_order
hashing_inputs_of_tasks
using_task_returns
+delayed_tasks
writing_custom_nodes
how_to_write_a_plugin
the_data_catalog
diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md
index cfb340a4..2c636579 100644
--- a/docs/source/tutorials/defining_dependencies_products.md
+++ b/docs/source/tutorials/defining_dependencies_products.md
@@ -410,6 +410,8 @@ def task_fit_model(depends_on, produces):
:::
::::
+(after)=
+
## Depending on a task
In some situations you want to define a task depending on another task without
diff --git a/docs_src/how_to_guides/delayed_tasks_delayed_products.py b/docs_src/how_to_guides/delayed_tasks_delayed_products.py
new file mode 100644
index 00000000..2e85b05a
--- /dev/null
+++ b/docs_src/how_to_guides/delayed_tasks_delayed_products.py
@@ -0,0 +1,25 @@
+from pathlib import Path
+
+import requests
+from pytask import DelayedPathNode
+from pytask import Product
+from typing_extensions import Annotated
+
+
+def task_download_files(
+ download_folder: Annotated[
+ Path, DelayedPathNode(root_dir=Path("downloads"), pattern="*"), Product
+ ],
+) -> None:
+ """Download files."""
+ # Scrape list of files without file extension from
+ # https://github.com/pytask-dev/pytask. (We skip this part for simplicity.)
+ files_to_download = ("CITATION", "LICENSE")
+
+ # Download them.
+ for file_ in files_to_download:
+ response = requests.get(
+ url=f"raw.githubusercontent.com/pytask-dev/pytask/main/{file_}", timeout=5
+ )
+ content = response.text
+ download_folder.joinpath(file_).write_text(content)
diff --git a/docs_src/how_to_guides/delayed_tasks_delayed_task.py b/docs_src/how_to_guides/delayed_tasks_delayed_task.py
new file mode 100644
index 00000000..7c56e14d
--- /dev/null
+++ b/docs_src/how_to_guides/delayed_tasks_delayed_task.py
@@ -0,0 +1,14 @@
+from pathlib import Path
+
+from pytask import DelayedPathNode
+from typing_extensions import Annotated
+
+
+def task_merge_files(
+ paths: Annotated[
+ list[Path], DelayedPathNode(root_dir=Path("downloads"), pattern="*")
+ ],
+) -> Annotated[str, Path("all_text.txt")]:
+ """Merge files."""
+ contents = [path.read_text() for path in paths]
+ return "\n".join(contents)
diff --git a/docs_src/how_to_guides/delayed_tasks_task_generator.py b/docs_src/how_to_guides/delayed_tasks_task_generator.py
new file mode 100644
index 00000000..f5753584
--- /dev/null
+++ b/docs_src/how_to_guides/delayed_tasks_task_generator.py
@@ -0,0 +1,21 @@
+from pathlib import Path
+
+from pytask import DelayedPathNode
+from pytask import task
+from typing_extensions import Annotated
+
+
+@task(generator=True)
+def task_copy_files(
+ paths: Annotated[
+ list[Path], DelayedPathNode(root_dir=Path("downloads"), pattern="*")
+ ],
+) -> None:
+ """Create tasks to copy each file to a ``.txt`` file."""
+ for path in paths:
+ # The path of the copy will be CITATION.txt, for example.
+ path_to_copy = path.with_suffix(".txt")
+
+ @task
+ def copy_file(path: Annotated[Path, path]) -> Annotated[str, path_to_copy]:
+ return path.read_text()
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 0c7b026f..48e3cba7 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -261,7 +261,7 @@ def pytask_collect_task(
collection_id = obj.pytask_meta._id if hasattr(obj, "pytask_meta") else None
after = obj.pytask_meta.after if hasattr(obj, "pytask_meta") else []
is_generator = (
- obj.pytask_meta.generator if hasattr(obj, "pytask_meta") else False
+ obj.pytask_meta.is_generator if hasattr(obj, "pytask_meta") else False
)
# Get the underlying function to avoid having different states of the function,
diff --git a/src/_pytask/models.py b/src/_pytask/models.py
index 2babee1c..91a2a915 100644
--- a/src/_pytask/models.py
+++ b/src/_pytask/models.py
@@ -31,6 +31,8 @@ class CollectionMetadata:
id will be generated. See
:doc:`this tutorial <../tutorials/repeating_tasks_with_different_inputs>` for
more information.
+ is_generator
+ An indicator for whether a task generates other tasks or not.
kwargs
A dictionary containing keyword arguments which are passed to the task when it
is executed.
diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py
index 297f8505..7fd19035 100644
--- a/src/_pytask/task_utils.py
+++ b/src/_pytask/task_utils.py
@@ -113,7 +113,7 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
if hasattr(unwrapped, "pytask_meta"):
unwrapped.pytask_meta.after = parsed_after
- unwrapped.pytask_meta.generator = generator
+ unwrapped.pytask_meta.is_generator = generator
unwrapped.pytask_meta.id_ = id
unwrapped.pytask_meta.kwargs = parsed_kwargs
unwrapped.pytask_meta.markers.append(Mark("task", (), {}))
diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py
index 632d410f..3d4946c9 100644
--- a/tests/test_task_utils.py
+++ b/tests/test_task_utils.py
@@ -67,7 +67,7 @@ def task_example():
...
assert task_example.pytask_meta.after == []
- assert not task_example.pytask_meta.generator
+ assert not task_example.pytask_meta.is_generator
assert task_example.pytask_meta.id_ is None
assert task_example.pytask_meta.kwargs == {}
assert task_example.pytask_meta.markers == [Mark("task", (), {})]
From 4da17c82c9f6eeeb8a47d5300eacc11c57cb5827 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 09:38:02 +0100
Subject: [PATCH 48/81] Fix.
---
docs/source/how_to_guides/delayed_tasks.md | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/docs/source/how_to_guides/delayed_tasks.md b/docs/source/how_to_guides/delayed_tasks.md
index 4783220f..3bf7c125 100644
--- a/docs/source/how_to_guides/delayed_tasks.md
+++ b/docs/source/how_to_guides/delayed_tasks.md
@@ -1,6 +1,6 @@
# Delayed tasks
-pytask's execution model is can usually be viewed as three phases.
+pytask's execution model can usually be separated into three phases.
1. Collection of tasks, dependencies, and products.
1. Building the DAG.
@@ -8,9 +8,9 @@ pytask's execution model is can usually be viewed as three phases.
But, in some situations pytask needs to be more flexible.
-Imagine you want to download files from some online storage, but the total number of
-files and their filenames is unknown before the task has started. How can you describe
-the files still as products of the task?
+Imagine you want to download files from an online storage, but the total number of files
+and their filenames is unknown before the task has started. How can you describe the
+files still as products of the task?
And how would you define a task that depends on these files. Or, how would define a
single task to process each file.
From f1729929057ee8a9de92fa9fe9ae528ab2cfadd7 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 11:25:59 +0100
Subject: [PATCH 49/81] Rename delayed nodes to provisional nodes.
---
src/_pytask/collect.py | 6 +++---
src/_pytask/collect_utils.py | 4 ++--
src/_pytask/delayed.py | 12 ++++++------
src/_pytask/delayed_utils.py | 28 ++++++++++++++--------------
src/_pytask/execute.py | 14 ++++++++------
src/_pytask/node_protocols.py | 30 +++++++++++-------------------
src/_pytask/nodes.py | 4 ++--
src/_pytask/persist.py | 4 ++--
src/_pytask/skipping.py | 4 ++--
src/pytask/__init__.py | 4 ++--
10 files changed, 52 insertions(+), 58 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 48e3cba7..5e2d0a5e 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -25,9 +25,9 @@
from _pytask.mark import MarkGenerator
from _pytask.mark_utils import get_all_marks
from _pytask.mark_utils import has_mark
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.nodes import DelayedPathNode
from _pytask.nodes import PathNode
@@ -306,7 +306,7 @@ def pytask_collect_task(
@hookimpl(trylast=True)
def pytask_collect_node( # noqa: C901, PLR0912
session: Session, path: Path, node_info: NodeInfo
-) -> PNode | PDelayedNode:
+) -> PNode | PProvisionalNode:
"""Collect a node of a task as a :class:`pytask.PNode`.
Strings are assumed to be paths. This might be a strict assumption, but since this
@@ -338,7 +338,7 @@ def pytask_collect_node( # noqa: C901, PLR0912
)
node.name = Path(short_root_dir, node.pattern).as_posix()
- if isinstance(node, PDelayedNode):
+ if isinstance(node, PProvisionalNode):
return node
if isinstance(node, PythonNode):
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index 80c3af18..a986f0dc 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -18,8 +18,8 @@
from _pytask.mark_utils import has_mark
from _pytask.mark_utils import remove_marks
from _pytask.models import NodeInfo
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.nodes import PythonNode
from _pytask.shared import find_duplicates
from _pytask.task_utils import parse_keyword_arguments_from_signature_defaults
@@ -480,7 +480,7 @@ def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
task_path: Path | None,
parameter_name: str,
value: Any,
-) -> PyTree[PDelayedNode | PNode]:
+) -> PyTree[PProvisionalNode | PNode]:
return tree_map_with_path(
lambda p, x: collection_func(
session,
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 062d8d30..f5a0313d 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -8,9 +8,9 @@
from typing import TYPE_CHECKING
from _pytask.config import hookimpl
-from _pytask.delayed_utils import collect_delayed_nodes
+from _pytask.delayed_utils import collect_provisional_nodes
from _pytask.delayed_utils import recreate_dag
-from _pytask.delayed_utils import TASKS_WITH_DELAYED_NODES
+from _pytask.delayed_utils import TASKS_WITH_PROVISIONAL_NODES
from _pytask.exceptions import NodeLoadError
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
@@ -31,11 +31,11 @@
@hookimpl
def pytask_execute_task_setup(session: Session, task: PTask) -> None:
- """Collect delayed nodes and parse them."""
+ """Collect provisional nodes and parse them."""
task.depends_on = tree_map_with_path( # type: ignore[assignment]
- lambda p, x: collect_delayed_nodes(session, task, x, p), task.depends_on
+ lambda p, x: collect_provisional_nodes(session, task, x, p), task.depends_on
)
- if task.signature in TASKS_WITH_DELAYED_NODES:
+ if task.signature in TASKS_WITH_PROVISIONAL_NODES:
recreate_dag(session, task)
@@ -140,4 +140,4 @@ def pytask_execute_task_process_report(report: ExecutionReport) -> bool | None:
@hookimpl
def pytask_unconfigure() -> None:
"""Clear the global variable after execution."""
- TASKS_WITH_DELAYED_NODES.clear()
+ TASKS_WITH_PROVISIONAL_NODES.clear()
diff --git a/src/_pytask/delayed_utils.py b/src/_pytask/delayed_utils.py
index ffcbfc55..b04c64b1 100644
--- a/src/_pytask/delayed_utils.py
+++ b/src/_pytask/delayed_utils.py
@@ -8,8 +8,8 @@
from _pytask.collect_utils import collect_dependency
from _pytask.dag_utils import TopologicalSorter
from _pytask.models import NodeInfo
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
from _pytask.nodes import Task
@@ -22,26 +22,26 @@
from _pytask.session import Session
-TASKS_WITH_DELAYED_NODES = set()
+TASKS_WITH_PROVISIONAL_NODES = set()
-def collect_delayed_nodes(
+def collect_provisional_nodes(
session: Session, task: PTask, node: Any, path: tuple[Any, ...]
) -> PyTree[PNode]:
- """Collect delayed nodes.
+ """Collect provisional nodes.
1. Call the :meth:`pytask.PDelayedNode.collect` to receive the raw nodes.
2. Collect the raw nodes as usual.
"""
- if not isinstance(node, PDelayedNode):
+ if not isinstance(node, PProvisionalNode):
return node
# Add task to register to update the DAG after the task is executed.
- TASKS_WITH_DELAYED_NODES.add(task.signature)
+ TASKS_WITH_PROVISIONAL_NODES.add(task.signature)
- # Collect delayed nodes and receive raw nodes.
- delayed_nodes = node.collect()
+ # Collect provisional nodes and receive raw nodes.
+ provisional_nodes = node.collect()
# Collect raw nodes.
node_path = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd()
@@ -62,7 +62,7 @@ def collect_delayed_nodes(
task_name=task_name,
),
),
- delayed_nodes,
+ provisional_nodes,
)
@@ -83,8 +83,8 @@ def recreate_dag(session: Session, task: PTask) -> None:
session.should_stop = True
-def collect_delayed_products(session: Session, task: PTask) -> None:
- """Collect delayed products.
+def collect_provisional_products(session: Session, task: PTask) -> None:
+ """Collect provisional products.
Unfortunately, this function needs to be called when a task finishes successfully
(skipped unchanged, persisted, etc..).
@@ -93,10 +93,10 @@ def collect_delayed_products(session: Session, task: PTask) -> None:
if is_task_generator(task):
return
- # Replace delayed nodes with their actually resolved nodes.
+ # Replace provisional nodes with their actually resolved nodes.
task.produces = tree_map_with_path( # type: ignore[assignment]
- lambda p, x: collect_delayed_nodes(session, task, x, p), task.produces
+ lambda p, x: collect_provisional_nodes(session, task, x, p), task.produces
)
- if task.signature in TASKS_WITH_DELAYED_NODES:
+ if task.signature in TASKS_WITH_PROVISIONAL_NODES:
recreate_dag(session, task)
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index fa621228..e3fec83c 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -20,15 +20,15 @@
from _pytask.dag_utils import TopologicalSorter
from _pytask.database_utils import has_node_changed
from _pytask.database_utils import update_states_in_database
-from _pytask.delayed_utils import collect_delayed_products
+from _pytask.delayed_utils import collect_provisional_products
from _pytask.exceptions import ExecutionError
from _pytask.exceptions import NodeLoadError
from _pytask.exceptions import NodeNotFoundError
from _pytask.mark import Mark
from _pytask.mark_utils import has_mark
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.outcomes import count_outcomes
from _pytask.outcomes import Exit
@@ -155,7 +155,9 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C
raise NodeNotFoundError(msg)
# Skip delayed nodes that are products since they do not have a state.
- if node_signature not in predecessors and isinstance(node, PDelayedNode):
+ if node_signature not in predecessors and isinstance(
+ node, PProvisionalNode
+ ):
continue
has_changed = has_node_changed(task=task, node=node)
@@ -164,7 +166,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C
break
if not needs_to_be_executed:
- collect_delayed_products(session, task)
+ collect_provisional_products(session, task)
raise SkippedUnchanged
# Create directory for product if it does not exist. Maybe this should be a `setup`
@@ -217,7 +219,7 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
nodes = tree_leaves(task.produces["return"])
values = structure_return.flatten_up_to(out)
for node, value in zip(nodes, values):
- if not isinstance(node, PDelayedNode):
+ if not isinstance(node, PProvisionalNode):
node.save(value)
return True
@@ -229,7 +231,7 @@ def pytask_execute_task_teardown(session: Session, task: PTask) -> None:
if is_task_generator(task):
return
- collect_delayed_products(session, task)
+ collect_provisional_products(session, task)
missing_nodes = [node for node in tree_leaves(task.produces) if not node.state()]
if missing_nodes:
paths = session.config["paths"]
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index 95fa764e..c60bab9e 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -14,7 +14,7 @@
from _pytask.mark import Mark
-__all__ = ["PDelayedNode", "PNode", "PPathNode", "PTask", "PTaskWithPath"]
+__all__ = ["PNode", "PPathNode", "PProvisionalNode", "PTask", "PTaskWithPath"]
@runtime_checkable
@@ -105,24 +105,24 @@ class PTaskWithPath(PTask, Protocol):
@runtime_checkable
-class PDelayedNode(PNode, Protocol):
- """A protocol for delayed nodes.
+class PProvisionalNode(PNode, Protocol):
+ """A protocol for provisional nodes.
- Delayed nodes are called delayed because they are resolved to actual nodes,
- :class:`PNode`, right before a task is executed as a dependency and after the task
- is executed as a product.
+ This type of nodes is provisional since it resolves to actual nodes, :class:`PNode`,
+ right before a task is executed as a dependency and after the task is executed as a
+ product.
- Delayed nodes are nodes that define how the actual nodes look like. They can be
+ Provisional nodes are nodes that define how the actual nodes look like. They can be
useful when, for example, a task produces an unknown amount of nodes because it
downloads some files.
"""
def load(self, is_product: bool = False) -> Any:
- """The load method of a delayed node.
+ """The load method of a probisional node.
- A delayed node will never be loaded as a dependency since it would be collected
- before.
+ A provisional node will never be loaded as a dependency since it would be
+ collected before.
It is possible to load a delayed node as a dependency so that it can inject
basic information about it in the task. For example,
@@ -133,13 +133,5 @@ def load(self, is_product: bool = False) -> Any:
...
raise NotImplementedError
- def save(self, value: Any) -> None:
- """A delayed node can never save a value."""
- raise NotImplementedError
-
- def state(self) -> None:
- """A delayed node has not state."""
- raise NotImplementedError
-
def collect(self) -> list[Any]:
- """Collect the objects that are defined by the delayed nodes."""
+ """Collect the objects that are defined by the provisional nodes."""
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index c6358c02..2b23019e 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -10,9 +10,9 @@
from typing import TYPE_CHECKING
from _pytask._hashlib import hash_value
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
from _pytask.path import hash_path
@@ -334,7 +334,7 @@ def save(self, value: Any) -> None:
@define(kw_only=True)
-class DelayedPathNode(PDelayedNode):
+class DelayedPathNode(PProvisionalNode):
"""A class for delayed :class:`PathNode`.
Attributes
diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py
index 54de6b28..60e539cf 100644
--- a/src/_pytask/persist.py
+++ b/src/_pytask/persist.py
@@ -8,7 +8,7 @@
from _pytask.dag_utils import node_and_neighbors
from _pytask.database_utils import has_node_changed
from _pytask.database_utils import update_states_in_database
-from _pytask.delayed_utils import collect_delayed_products
+from _pytask.delayed_utils import collect_provisional_products
from _pytask.mark_utils import has_mark
from _pytask.outcomes import Persisted
from _pytask.outcomes import TaskOutcome
@@ -57,7 +57,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
for name in node_and_neighbors(session.dag, task.signature)
)
if any_node_changed:
- collect_delayed_products(session, task)
+ collect_provisional_products(session, task)
raise Persisted
diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py
index 18b50554..f1f17135 100644
--- a/src/_pytask/skipping.py
+++ b/src/_pytask/skipping.py
@@ -6,7 +6,7 @@
from _pytask.config import hookimpl
from _pytask.dag_utils import descending_tasks
-from _pytask.delayed_utils import collect_delayed_products
+from _pytask.delayed_utils import collect_provisional_products
from _pytask.mark import Mark
from _pytask.mark_utils import get_marks
from _pytask.mark_utils import has_mark
@@ -53,7 +53,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
task, "would_be_executed"
)
if is_unchanged and not session.config["force"]:
- collect_delayed_products(session, task)
+ collect_provisional_products(session, task)
raise SkippedUnchanged
is_skipped = has_mark(task, "skip")
diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py
index ec031d4b..ab100e95 100644
--- a/src/pytask/__init__.py
+++ b/src/pytask/__init__.py
@@ -39,9 +39,9 @@
from _pytask.mark_utils import set_marks
from _pytask.models import CollectionMetadata
from _pytask.models import NodeInfo
-from _pytask.node_protocols import PDelayedNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
from _pytask.nodes import DelayedPathNode
@@ -102,9 +102,9 @@
"NodeInfo",
"NodeNotCollectedError",
"NodeNotFoundError",
- "PDelayedNode",
"PNode",
"PPathNode",
+ "PProvisionalNode",
"PTask",
"PTaskWithPath",
"PathNode",
From 6cc4a28fee925f4beed21f06afa821b1fd364500 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 11:41:09 +0100
Subject: [PATCH 50/81] Make PDelayedNode independent from PNode.
---
src/_pytask/delayed_utils.py | 2 +-
src/_pytask/node_protocols.py | 14 ++++++++++----
src/_pytask/nodes.py | 14 ++++----------
3 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/src/_pytask/delayed_utils.py b/src/_pytask/delayed_utils.py
index b04c64b1..442d2d59 100644
--- a/src/_pytask/delayed_utils.py
+++ b/src/_pytask/delayed_utils.py
@@ -27,7 +27,7 @@
def collect_provisional_nodes(
session: Session, task: PTask, node: Any, path: tuple[Any, ...]
-) -> PyTree[PNode]:
+) -> PyTree[PNode | PProvisionalNode]:
"""Collect provisional nodes.
1. Call the :meth:`pytask.PDelayedNode.collect` to receive the raw nodes.
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index c60bab9e..81933366 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -69,8 +69,8 @@ class PTask(Protocol):
"""Protocol for nodes."""
name: str
- depends_on: dict[str, PyTree[PNode]]
- produces: dict[str, PyTree[PNode]]
+ depends_on: dict[str, PyTree[PNode | PProvisionalNode]]
+ produces: dict[str, PyTree[PNode | PProvisionalNode]]
function: Callable[..., Any]
markers: list[Mark]
report_sections: list[tuple[str, str, str]]
@@ -105,7 +105,7 @@ class PTaskWithPath(PTask, Protocol):
@runtime_checkable
-class PProvisionalNode(PNode, Protocol):
+class PProvisionalNode(Protocol):
"""A protocol for provisional nodes.
This type of nodes is provisional since it resolves to actual nodes, :class:`PNode`,
@@ -118,13 +118,19 @@ class PProvisionalNode(PNode, Protocol):
"""
+ name: str
+
+ @property
+ def signature(self) -> str:
+ """Return the signature of the node."""
+
def load(self, is_product: bool = False) -> Any:
"""The load method of a probisional node.
A provisional node will never be loaded as a dependency since it would be
collected before.
- It is possible to load a delayed node as a dependency so that it can inject
+ It is possible to load a provisional node as a dependency so that it can inject
basic information about it in the task. For example,
:meth:`pytask.DelayedPathNode` injects the root directory.
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 2b23019e..9521babc 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -60,8 +60,8 @@ class TaskWithoutPath(PTask):
name: str
function: Callable[..., Any]
- depends_on: dict[str, PyTree[PNode]] = field(factory=dict)
- produces: dict[str, PyTree[PNode]] = field(factory=dict)
+ depends_on: dict[str, PyTree[PNode | PProvisionalNode]] = field(factory=dict)
+ produces: dict[str, PyTree[PNode | PProvisionalNode]] = field(factory=dict)
markers: list[Mark] = field(factory=list)
report_sections: list[tuple[str, str, str]] = field(factory=list)
attributes: dict[Any, Any] = field(factory=dict)
@@ -116,8 +116,8 @@ class Task(PTaskWithPath):
path: Path
function: Callable[..., Any]
name: str = field(default="", init=False)
- depends_on: dict[str, PyTree[PNode]] = field(factory=dict)
- produces: dict[str, PyTree[PNode]] = field(factory=dict)
+ depends_on: dict[str, PyTree[PNode | PProvisionalNode]] = field(factory=dict)
+ produces: dict[str, PyTree[PNode | PProvisionalNode]] = field(factory=dict)
markers: list[Mark] = field(factory=list)
report_sections: list[tuple[str, str, str]] = field(factory=list)
attributes: dict[Any, Any] = field(factory=dict)
@@ -365,12 +365,6 @@ def load(self, is_product: bool = False) -> Path:
return self.root_dir # type: ignore[return-value]
raise NotImplementedError
- def save(self, value: Any) -> None:
- raise NotImplementedError
-
- def state(self) -> None:
- raise NotImplementedError
-
def collect(self) -> list[Path]:
"""Collect paths defined by the pattern."""
return list(self.root_dir.glob(self.pattern)) # type: ignore[union-attr]
From bfd7d38c61bcc7a8ee3846e9f10cee34bb703a91 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 11:48:28 +0100
Subject: [PATCH 51/81] Rename DelayedPathNode to DirectoryNode.
---
docs/source/how_to_guides/delayed_tasks.md | 12 ++--
docs/source/reference_guides/api.md | 4 +-
.../delayed_tasks_delayed_products.py | 4 +-
.../delayed_tasks_delayed_task.py | 4 +-
.../delayed_tasks_task_generator.py | 4 +-
src/_pytask/collect.py | 4 +-
src/_pytask/delayed_utils.py | 2 +-
src/_pytask/node_protocols.py | 2 +-
src/_pytask/nodes.py | 22 ++++----
src/pytask/__init__.py | 4 +-
tests/test_collect_command.py | 12 ++--
tests/test_execute.py | 56 +++++++++----------
12 files changed, 65 insertions(+), 65 deletions(-)
diff --git a/docs/source/how_to_guides/delayed_tasks.md b/docs/source/how_to_guides/delayed_tasks.md
index 3bf7c125..16d0915e 100644
--- a/docs/source/how_to_guides/delayed_tasks.md
+++ b/docs/source/how_to_guides/delayed_tasks.md
@@ -30,13 +30,13 @@ emphasize-lines: 4, 11
```
Since the names of the filesare not known when pytask is started, we need to use a
-{class}`~pytask.DelayedPathNode`. With a {class}`~pytask.DelayedPathNode` we can specify
+{class}`~pytask.DirectoryNode`. With a {class}`~pytask.DirectoryNode` we can specify
where pytask can find the files and how they look like with an optional path and a glob
pattern.
-When we use the {class}`~pytask.DelayedPathNode` as a product annotation, we get access
-to the `root_dir` as a {class}`~pathlib.Path` object inside the function which allows us
-to store the files.
+When we use the {class}`~pytask.DirectoryNode` as a product annotation, we get access to
+the `root_dir` as a {class}`~pathlib.Path` object inside the function which allows us to
+store the files.
## Delayed task
@@ -49,11 +49,11 @@ emphasize-lines: 8-10
---
```
-When {class}`~pytask.DelayedPathNode` is used as a dependency a list of all the files in
+When {class}`~pytask.DirectoryNode` is used as a dependency a list of all the files in
the folder defined by the root path and the pattern are automatically collected and
passed to the task.
-As long as we use a {class}`DelayedPathNode` with the same `root_dir` and `pattern` in
+As long as we use a {class}`DirectoryNode` with the same `root_dir` and `pattern` in
both tasks, pytask will automatically recognize that the second task depends on the
first. If that is not true, you might need to make this dependency more explicit by
using {func}`@task(after=...) <pytask.task>` which is explained {ref}`here <after>`.
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index d6230442..bd2bfbdd 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -253,7 +253,7 @@ Protocols define how tasks and nodes for dependencies and products have to be se
:show-inheritance:
.. autoprotocol:: pytask.PTaskWithPath
:show-inheritance:
-.. autoprotocol:: pytask.PDelayedNode
+.. autoprotocol:: pytask.PProvisionalNode
:show-inheritance:
```
@@ -268,7 +268,7 @@ Nodes are the interface for different kinds of dependencies or products.
:members: load, save
.. autoclass:: pytask.PythonNode
:members: load, save
-.. autoclass:: pytask.DelayedPathNode
+.. autoclass:: pytask.DirectoryNode
:members: load, collect
```
diff --git a/docs_src/how_to_guides/delayed_tasks_delayed_products.py b/docs_src/how_to_guides/delayed_tasks_delayed_products.py
index 2e85b05a..5b6454f5 100644
--- a/docs_src/how_to_guides/delayed_tasks_delayed_products.py
+++ b/docs_src/how_to_guides/delayed_tasks_delayed_products.py
@@ -1,14 +1,14 @@
from pathlib import Path
import requests
-from pytask import DelayedPathNode
+from pytask import DirectoryNode
from pytask import Product
from typing_extensions import Annotated
def task_download_files(
download_folder: Annotated[
- Path, DelayedPathNode(root_dir=Path("downloads"), pattern="*"), Product
+ Path, DirectoryNode(root_dir=Path("downloads"), pattern="*"), Product
],
) -> None:
"""Download files."""
diff --git a/docs_src/how_to_guides/delayed_tasks_delayed_task.py b/docs_src/how_to_guides/delayed_tasks_delayed_task.py
index 7c56e14d..1f158eef 100644
--- a/docs_src/how_to_guides/delayed_tasks_delayed_task.py
+++ b/docs_src/how_to_guides/delayed_tasks_delayed_task.py
@@ -1,12 +1,12 @@
from pathlib import Path
-from pytask import DelayedPathNode
+from pytask import DirectoryNode
from typing_extensions import Annotated
def task_merge_files(
paths: Annotated[
- list[Path], DelayedPathNode(root_dir=Path("downloads"), pattern="*")
+ list[Path], DirectoryNode(root_dir=Path("downloads"), pattern="*")
],
) -> Annotated[str, Path("all_text.txt")]:
"""Merge files."""
diff --git a/docs_src/how_to_guides/delayed_tasks_task_generator.py b/docs_src/how_to_guides/delayed_tasks_task_generator.py
index f5753584..e2390d4e 100644
--- a/docs_src/how_to_guides/delayed_tasks_task_generator.py
+++ b/docs_src/how_to_guides/delayed_tasks_task_generator.py
@@ -1,6 +1,6 @@
from pathlib import Path
-from pytask import DelayedPathNode
+from pytask import DirectoryNode
from pytask import task
from typing_extensions import Annotated
@@ -8,7 +8,7 @@
@task(generator=True)
def task_copy_files(
paths: Annotated[
- list[Path], DelayedPathNode(root_dir=Path("downloads"), pattern="*")
+ list[Path], DirectoryNode(root_dir=Path("downloads"), pattern="*")
],
) -> None:
"""Create tasks to copy each file to a ``.txt`` file."""
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 5e2d0a5e..0aa049e3 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -29,7 +29,7 @@
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
-from _pytask.nodes import DelayedPathNode
+from _pytask.nodes import DirectoryNode
from _pytask.nodes import PathNode
from _pytask.nodes import PythonNode
from _pytask.nodes import Task
@@ -326,7 +326,7 @@ def pytask_collect_node( # noqa: C901, PLR0912
"""
node = node_info.value
- if isinstance(node, DelayedPathNode):
+ if isinstance(node, DirectoryNode):
if node.root_dir is None:
node.root_dir = path
if (
diff --git a/src/_pytask/delayed_utils.py b/src/_pytask/delayed_utils.py
index 442d2d59..e01f39b2 100644
--- a/src/_pytask/delayed_utils.py
+++ b/src/_pytask/delayed_utils.py
@@ -30,7 +30,7 @@ def collect_provisional_nodes(
) -> PyTree[PNode | PProvisionalNode]:
"""Collect provisional nodes.
- 1. Call the :meth:`pytask.PDelayedNode.collect` to receive the raw nodes.
+ 1. Call the :meth:`pytask.PProvisionalNode.collect` to receive the raw nodes.
2. Collect the raw nodes as usual.
"""
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index 81933366..e57dbd8b 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -132,7 +132,7 @@ def load(self, is_product: bool = False) -> Any:
It is possible to load a provisional node as a dependency so that it can inject
basic information about it in the task. For example,
- :meth:`pytask.DelayedPathNode` injects the root directory.
+ :meth:`pytask.DirectoryNode` injects the root directory.
"""
if is_product:
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 9521babc..7b5eb75d 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -28,7 +28,7 @@
from _pytask.mark import Mark
-__all__ = ["DelayedPathNode", "PathNode", "PythonNode", "Task", "TaskWithoutPath"]
+__all__ = ["DirectoryNode", "PathNode", "PythonNode", "Task", "TaskWithoutPath"]
@define(kw_only=True)
@@ -287,7 +287,7 @@ def state(self) -> str | None:
@define
-class PickleNode:
+class PickleNode(PNode):
"""A node for pickle files.
Attributes
@@ -334,25 +334,25 @@ def save(self, value: Any) -> None:
@define(kw_only=True)
-class DelayedPathNode(PProvisionalNode):
- """A class for delayed :class:`PathNode`.
+class DirectoryNode(PProvisionalNode):
+ """The class for a provisional node that works with directories.
Attributes
----------
- root_dir
- The pattern is interpreted relative to the path given by ``root_dir``. If
- ``root_dir = None``, it is the directory where the path is defined.
+ name
+ The name of the node.
pattern
Patterns are the same as for :mod:`fnmatch`, with the addition of ``**`` which
means "this directory and all subdirectories, recursively".
- name
- The name of the node.
+ root_dir
+ The pattern is interpreted relative to the path given by ``root_dir``. If
+ ``root_dir = None``, it is the directory where the path is defined.
"""
- root_dir: Path | None = None
- pattern: str
name: str = ""
+ pattern: str = "*"
+ root_dir: Path | None = None
@property
def signature(self) -> str:
diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py
index ab100e95..8d1191db 100644
--- a/src/pytask/__init__.py
+++ b/src/pytask/__init__.py
@@ -44,7 +44,7 @@
from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
-from _pytask.nodes import DelayedPathNode
+from _pytask.nodes import DirectoryNode
from _pytask.nodes import PathNode
from _pytask.nodes import PickleNode
from _pytask.nodes import PythonNode
@@ -90,7 +90,7 @@
"DagReport",
"DataCatalog",
"DatabaseSession",
- "DelayedPathNode",
+ "DirectoryNode",
"EnumChoice",
"ExecutionError",
"ExecutionReport",
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 5389d5ad..6d831121 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -666,14 +666,14 @@ def task_example() -> Annotated[Dict[str, str], nodes]:
@pytest.mark.parametrize(
"node_def",
[
- "paths: Annotated[List[Path], DelayedPathNode(pattern='*.txt'), Product])",
- "produces=DelayedPathNode(pattern='*.txt'))",
- ") -> Annotated[None, DelayedPathNode(pattern='*.txt')]",
+ "paths: Annotated[List[Path], DirectoryNode(pattern='*.txt'), Product])",
+ "produces=DirectoryNode(pattern='*.txt'))",
+ ") -> Annotated[None, DirectoryNode(pattern='*.txt')]",
],
)
def test_collect_task_with_delayed_path_node_as_product(runner, tmp_path, node_def):
source = f"""
- from pytask import DelayedPathNode, Product
+ from pytask import DirectoryNode, Product
from typing_extensions import Annotated, List
from pathlib import Path
@@ -705,11 +705,11 @@ def task_example({node_def}: ...
def test_collect_task_with_delayed_dependencies(runner, tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode
+ from pytask import DirectoryNode
from pathlib import Path
def task_example(
- paths = DelayedPathNode(pattern="[ab].txt")
+ paths = DirectoryNode(pattern="[ab].txt")
) -> Annotated[str, Path("merged.txt")]:
path_dict = {path.stem: path for path in paths}
return path_dict["a"].read_text() + path_dict["b"].read_text()
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 2ed85b85..f1fad2fc 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1054,11 +1054,11 @@ def func(path): path.touch()
def test_task_that_produces_delayed_path_node(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, Product
+ from pytask import DirectoryNode, Product
from pathlib import Path
def task_example(
- root_path: Annotated[Path, DelayedPathNode(pattern="*.txt"), Product]
+ root_path: Annotated[Path, DirectoryNode(pattern="*.txt"), Product]
):
root_path.joinpath("a.txt").write_text("Hello, ")
root_path.joinpath("b.txt").write_text("World!")
@@ -1080,11 +1080,11 @@ def task_example(
def test_task_that_depends_on_relative_delayed_path_node(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode
+ from pytask import DirectoryNode
from pathlib import Path
def task_example(
- paths = DelayedPathNode(pattern="[ab].txt")
+ paths = DirectoryNode(pattern="[ab].txt")
) -> Annotated[str, Path("merged.txt")]:
path_dict = {path.stem: path for path in paths}
return path_dict["a"].read_text() + path_dict["b"].read_text()
@@ -1104,13 +1104,13 @@ def task_example(
def test_task_that_depends_on_delayed_path_node_with_root_dir(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode
+ from pytask import DirectoryNode
from pathlib import Path
root_dir = Path(__file__).parent / "subfolder"
def task_example(
- paths = DelayedPathNode(root_dir=root_dir, pattern="[ab].txt")
+ paths = DirectoryNode(root_dir=root_dir, pattern="[ab].txt")
) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
path_dict = {path.stem: path for path in paths}
return path_dict["a"].read_text() + path_dict["b"].read_text()
@@ -1131,17 +1131,17 @@ def task_example(
def test_task_that_depends_on_delayed_task(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task
+ from pytask import DirectoryNode, task
from pathlib import Path
- def task_produces() -> Annotated[None, DelayedPathNode(pattern="[ab].txt")]:
+ def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
@task(after=task_produces)
def task_depends(
- paths = DelayedPathNode(pattern="[ab].txt")
+ paths = DirectoryNode(pattern="[ab].txt")
) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
path_dict = {path.stem: path for path in paths}
return path_dict["a"].read_text() + path_dict["b"].read_text()
@@ -1160,17 +1160,17 @@ def task_depends(
def test_gracefully_fail_when_dag_raises_error(runner, tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task
+ from pytask import DirectoryNode, task
from pathlib import Path
- def task_produces() -> Annotated[None, DelayedPathNode(pattern="*.txt")]:
+ def task_produces() -> Annotated[None, DirectoryNode(pattern="*.txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
@task(after=task_produces)
def task_depends(
- paths = DelayedPathNode(pattern="[ab].txt")
+ paths = DirectoryNode(pattern="[ab].txt")
) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]:
path_dict = {path.stem: path for path in paths}
return path_dict["a"].read_text() + path_dict["b"].read_text()
@@ -1189,17 +1189,17 @@ def task_depends(
def test_delayed_task_generation(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task
+ from pytask import DirectoryNode, task
from pathlib import Path
- def task_produces() -> Annotated[None, DelayedPathNode(pattern="[ab].txt")]:
+ def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
@task(after=task_produces, generator=True)
def task_depends(
- paths = DelayedPathNode(pattern="[ab].txt")
+ paths = DirectoryNode(pattern="[ab].txt")
) -> ...:
for path in paths:
@@ -1225,17 +1225,17 @@ def task_copy(
def test_delayed_task_generation_with_generator(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task
+ from pytask import DirectoryNode, task
from pathlib import Path
- def task_produces() -> Annotated[None, DelayedPathNode(pattern="[ab].txt")]:
+ def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
@task(after=task_produces, generator=True)
def task_depends(
- paths = DelayedPathNode(pattern="[ab].txt")
+ paths = DirectoryNode(pattern="[ab].txt")
) -> ...:
for path in paths:
@@ -1263,16 +1263,16 @@ def task_copy(
def test_delayed_task_generation_with_single_function(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task
+ from pytask import DirectoryNode, task
from pathlib import Path
- def task_produces() -> Annotated[None, DelayedPathNode(pattern="[a].txt")]:
+ def task_produces() -> Annotated[None, DirectoryNode(pattern="[a].txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
@task(after=task_produces, generator=True)
def task_depends(
- paths = DelayedPathNode(pattern="[a].txt")
+ paths = DirectoryNode(pattern="[a].txt")
) -> ...:
path = paths[0]
@@ -1297,16 +1297,16 @@ def task_copy(
def test_delayed_task_generation_with_task_node(tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, TaskWithoutPath, task, PathNode
+ from pytask import DirectoryNode, TaskWithoutPath, task, PathNode
from pathlib import Path
- def task_produces() -> Annotated[None, DelayedPathNode(pattern="[a].txt")]:
+ def task_produces() -> Annotated[None, DirectoryNode(pattern="[a].txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
@task(after=task_produces, generator=True)
def task_depends(
- paths = DelayedPathNode(pattern="[a].txt")
+ paths = DirectoryNode(pattern="[a].txt")
) -> ...:
path = paths[0]
@@ -1333,12 +1333,12 @@ def task_depends(
def test_gracefully_fail_when_task_generator_raises_error(runner, tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task, Product
+ from pytask import DirectoryNode, task, Product
from pathlib import Path
@task(generator=True)
def task_example(
- root_dir: Annotated[Path, DelayedPathNode(pattern="[a].txt"), Product]
+ root_dir: Annotated[Path, DirectoryNode(pattern="[a].txt"), Product]
) -> ...:
raise Exception
"""
@@ -1354,12 +1354,12 @@ def task_example(
def test_use_delayed_node_as_product_in_generator_without_rerun(runner, tmp_path):
source = """
from typing_extensions import Annotated
- from pytask import DelayedPathNode, task, Product
+ from pytask import DirectoryNode, task, Product
from pathlib import Path
@task(generator=True)
def task_example(
- root_dir: Annotated[Path, DelayedPathNode(pattern="[ab].txt"), Product]
+ root_dir: Annotated[Path, DirectoryNode(pattern="[ab].txt"), Product]
) -> ...:
for path in (root_dir / "a.txt", root_dir / "b.txt"):
From 80bd1ed5f2918e2d39d5247c846d42d266fe716c Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 12:27:14 +0100
Subject: [PATCH 52/81] Add test for data catalog.
---
src/_pytask/data_catalog.py | 11 +++++++----
tests/test_data_catalog.py | 28 ++++++++++++++++++++++++++++
2 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/src/_pytask/data_catalog.py b/src/_pytask/data_catalog.py
index 84c25375..564f4de5 100644
--- a/src/_pytask/data_catalog.py
+++ b/src/_pytask/data_catalog.py
@@ -15,6 +15,7 @@
from _pytask.models import NodeInfo
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.nodes import PickleNode
from _pytask.pluginmanager import get_plugin_manager
from _pytask.session import Session
@@ -64,7 +65,7 @@ class DataCatalog:
"""
default_node: type[PNode] = PickleNode
- entries: dict[str, PNode] = field(factory=dict)
+ entries: dict[str, PNode | PProvisionalNode] = field(factory=dict)
name: str = "default"
path: Path | None = None
_session: Session = field(factory=_create_default_session)
@@ -87,13 +88,15 @@ def _initialize(self) -> None:
node = pickle.loads(path.read_bytes()) # noqa: S301
self.entries[node.name] = node
- def __getitem__(self, name: str) -> DataCatalog | PNode:
+ def __getitem__(self, name: str) -> DataCatalog | PNode | PProvisionalNode:
"""Allow to access entries with the squared brackets syntax."""
if name not in self.entries:
self.add(name)
return self.entries[name]
- def add(self, name: str, node: DataCatalog | PNode | None = None) -> None:
+ def add(
+ self, name: str, node: DataCatalog | PNode | PProvisionalNode | None = None
+ ) -> None:
"""Add an entry to the data catalog."""
assert isinstance(self.path, Path)
@@ -112,7 +115,7 @@ def add(self, name: str, node: DataCatalog | PNode | None = None) -> None:
self.path.joinpath(f"{filename}-node.pkl").write_bytes(
pickle.dumps(self.entries[name])
)
- elif isinstance(node, PNode):
+ elif isinstance(node, (PNode, PProvisionalNode)):
self.entries[name] = node
else:
collected_node = self._session.hook.pytask_collect_node(
diff --git a/tests/test_data_catalog.py b/tests/test_data_catalog.py
index e54b8e1a..fe5daeae 100644
--- a/tests/test_data_catalog.py
+++ b/tests/test_data_catalog.py
@@ -207,3 +207,31 @@ def test_adding_a_python_node():
data_catalog = DataCatalog()
data_catalog.add("node", PythonNode(name="node", value=1))
assert isinstance(data_catalog["node"], PythonNode)
+
+
+@pytest.mark.end_to_end()
+def test_use_data_catalog_with_provisional_node(runner, tmp_path):
+ source = """
+ from pathlib import Path
+ from typing_extensions import Annotated, List
+
+ from pytask import DataCatalog
+ from pytask import DirectoryNode
+
+ # Generate input data
+ data_catalog = DataCatalog()
+ data_catalog.add("directory", DirectoryNode(pattern="*.txt"))
+
+ def task_add_content(
+ paths: Annotated[List[Path], data_catalog["directory"]]
+ ) -> Annotated[str, Path("output.txt")]:
+ name_to_path = {path.stem: path for path in paths}
+ return name_to_path["a"].read_text() + name_to_path["b"].read_text()
+ """
+ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
+ tmp_path.joinpath("a.txt").write_text("Hello, ")
+ tmp_path.joinpath("b.txt").write_text("World!")
+
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert tmp_path.joinpath("output.txt").read_text() == "Hello, World!"
From 23c7362f41f487b4a05216da656c719f1bb5faf8 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 12:36:09 +0100
Subject: [PATCH 53/81] Adjust more types.
---
src/_pytask/console.py | 5 ++++-
src/_pytask/dag.py | 9 +++++++--
src/_pytask/delayed.py | 3 ++-
src/_pytask/execute.py | 2 +-
4 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/src/_pytask/console.py b/src/_pytask/console.py
index b637b3ec..8bd5d33e 100644
--- a/src/_pytask/console.py
+++ b/src/_pytask/console.py
@@ -16,6 +16,7 @@
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTaskWithPath
from _pytask.path import shorten_path
from rich.console import Console
@@ -141,7 +142,9 @@ def format_task_name(task: PTask, editor_url_scheme: str) -> Text:
return Text(task.name, style=url_style)
-def format_node_name(node: PNode, paths: Sequence[Path] = ()) -> Text:
+def format_node_name(
+ node: PNode | PProvisionalNode, paths: Sequence[Path] = ()
+) -> Text:
"""Format the name of a node."""
if isinstance(node, PPathNode):
if node.name != node.path.as_posix():
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index 1278b8aa..5cf13416 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -17,6 +17,7 @@
from _pytask.exceptions import ResolvingDependenciesError
from _pytask.mark import select_by_after_keyword
from _pytask.node_protocols import PNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.nodes import PythonNode
from _pytask.reports import DagReport
@@ -54,7 +55,9 @@ def pytask_dag(session: Session) -> bool | None:
def pytask_dag_create_dag(session: Session, tasks: list[PTask]) -> nx.DiGraph:
"""Create the DAG from tasks, dependencies and products."""
- def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
+ def _add_dependency(
+ dag: nx.DiGraph, task: PTask, node: PNode | PProvisionalNode
+ ) -> None:
"""Add a dependency to the DAG."""
dag.add_node(node.signature, node=node)
dag.add_edge(node.signature, task.signature)
@@ -65,7 +68,9 @@ def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
if isinstance(node, PythonNode) and isinstance(node.value, PythonNode):
dag.add_edge(node.value.signature, node.signature)
- def _add_product(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
+ def _add_product(
+ dag: nx.DiGraph, task: PTask, node: PNode | PProvisionalNode
+ ) -> None:
"""Add a product to the DAG."""
dag.add_node(node.signature, node=node)
dag.add_edge(task.signature, node.signature)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index f5a0313d..2015d38c 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -13,6 +13,7 @@
from _pytask.delayed_utils import TASKS_WITH_PROVISIONAL_NODES
from _pytask.exceptions import NodeLoadError
from _pytask.node_protocols import PNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
from _pytask.outcomes import CollectionOutcome
@@ -39,7 +40,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
recreate_dag(session, task)
-def _safe_load(node: PNode, task: PTask, is_product: bool) -> Any:
+def _safe_load(node: PNode | PProvisionalNode, task: PTask, is_product: bool) -> Any:
try:
return node.load(is_product=is_product)
except Exception as e: # noqa: BLE001
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index e3fec83c..b5099042 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -177,7 +177,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C
node.path.parent.mkdir(parents=True, exist_ok=True)
-def _safe_load(node: PNode, task: PTask, is_product: bool) -> Any:
+def _safe_load(node: PNode | PProvisionalNode, task: PTask, is_product: bool) -> Any:
try:
return node.load(is_product=is_product)
except Exception as e: # noqa: BLE001
From 89e1014ec0c9921b7c4e1fc16f21bc48bf6479c6 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 12:43:09 +0100
Subject: [PATCH 54/81] FIx types.
---
src/_pytask/collect_utils.py | 13 +++++++++----
src/_pytask/dag.py | 2 +-
src/_pytask/hookspecs.py | 3 ++-
src/_pytask/reports.py | 5 +++--
src/_pytask/shared.py | 8 ++++++--
5 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index a986f0dc..f18df266 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -308,7 +308,10 @@ def parse_dependencies_from_task_function(
are_all_nodes_python_nodes_without_hash = all(
isinstance(x, PythonNode) and not x.hash for x in tree_leaves(nodes)
)
- if not isinstance(nodes, PNode) and are_all_nodes_python_nodes_without_hash:
+ if (
+ not isinstance(nodes, (PNode, PProvisionalNode))
+ and are_all_nodes_python_nodes_without_hash
+ ):
node_name = create_name_of_python_node(
NodeInfo(
arg_name=parameter_name,
@@ -324,7 +327,9 @@ def parse_dependencies_from_task_function(
return dependencies
-def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, PNode]:
+def _find_args_with_node_annotation(
+ func: Callable[..., Any]
+) -> dict[str, PNode | PProvisionalNode]:
"""Find args with node annotations."""
annotations = get_annotations(func, eval_str=True)
metas = {
@@ -568,7 +573,7 @@ def _collect_decorator_node(
def collect_dependency(
session: Session, path: Path, name: str, node_info: NodeInfo
-) -> PNode:
+) -> PNode | PProvisionalNode:
"""Collect nodes for a task.
Raises
@@ -612,7 +617,7 @@ def _collect_product(
path: Path,
task_name: str,
node_info: NodeInfo,
-) -> PNode:
+) -> PNode | PProvisionalNode:
"""Collect products for a task.
Defining products with strings is only allowed when using the decorator. Parameter
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index 5cf13416..555382ce 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -151,7 +151,7 @@ def _format_cycles(dag: nx.DiGraph, cycles: list[tuple[str, ...]]) -> str:
node = dag.nodes[x].get("task") or dag.nodes[x].get("node")
if isinstance(node, PTask):
short_name = format_task_name(node, editor_url_scheme="no_link").plain
- elif isinstance(node, PNode):
+ elif isinstance(node, (PNode, PProvisionalNode)):
short_name = node.name
lines.extend((short_name, " " + ARROW_DOWN_ICON))
# Join while removing last arrow.
diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py
index 8e944e9f..42ae10fa 100644
--- a/src/_pytask/hookspecs.py
+++ b/src/_pytask/hookspecs.py
@@ -13,6 +13,7 @@
if TYPE_CHECKING:
+ from _pytask.node_protocols import PProvisionalNode
from _pytask.models import NodeInfo
from _pytask.node_protocols import PNode
import click
@@ -196,7 +197,7 @@ def pytask_collect_task_teardown(session: Session, task: PTask) -> None:
@hookspec(firstresult=True)
def pytask_collect_node(
session: Session, path: Path, node_info: NodeInfo
-) -> PNode | None:
+) -> PNode | PProvisionalNode | None:
"""Collect a node which is a dependency or a product of a task."""
diff --git a/src/_pytask/reports.py b/src/_pytask/reports.py
index efde5ab9..611388e5 100644
--- a/src/_pytask/reports.py
+++ b/src/_pytask/reports.py
@@ -17,6 +17,7 @@
if TYPE_CHECKING:
+ from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PTask
from rich.console import Console
@@ -29,7 +30,7 @@ class CollectionReport:
"""A collection report for a task."""
outcome: CollectionOutcome
- node: PTask | PNode | None = None
+ node: PTask | PNode | PProvisionalNode | None = None
exc_info: OptionalExceptionInfo | None = None
@classmethod
@@ -37,7 +38,7 @@ def from_exception(
cls: type[CollectionReport],
outcome: CollectionOutcome,
exc_info: OptionalExceptionInfo,
- node: PTask | PNode | None = None,
+ node: PTask | PNode | PProvisionalNode | None = None,
) -> CollectionReport:
return cls(outcome=outcome, node=node, exc_info=exc_info)
diff --git a/src/_pytask/shared.py b/src/_pytask/shared.py
index 4afdc0ff..01080a21 100644
--- a/src/_pytask/shared.py
+++ b/src/_pytask/shared.py
@@ -12,6 +12,7 @@
from _pytask.console import format_node_name
from _pytask.console import format_task_name
from _pytask.node_protocols import PNode
+from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
if TYPE_CHECKING:
@@ -69,10 +70,13 @@ def reduce_names_of_multiple_nodes(
if isinstance(node, PTask):
short_name = format_task_name(node, editor_url_scheme="no_link").plain
- elif isinstance(node, PNode):
+ elif isinstance(node, (PNode, PProvisionalNode)):
short_name = format_node_name(node, paths).plain
else:
- msg = f"Requires a 'PTask' or a 'PNode' and not {type(node)!r}."
+ msg = (
+ "Requires a 'PTask', 'PNode', or 'PProvisionalNode' and not "
+ f"{type(node)!r}."
+ )
raise TypeError(msg)
short_names.append(short_name)
From c5eb99e32055d0150c62e56b7a8f098af15c87ec Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 13:09:17 +0100
Subject: [PATCH 55/81] Update docs.
---
.pre-commit-config.yaml | 1 +
docs/source/how_to_guides/delayed_tasks.md | 28 ++++++++++++++--------
2 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6d7527f8..8d6b781f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -99,6 +99,7 @@ repos:
docs/source/how_to_guides/using_task_returns.md|
docs/source/how_to_guides/writing_custom_nodes.md|
docs/source/how_to_guides/hashing_inputs_of_tasks.md|
+ docs/source/how_to_guides/delayed_tasks.md|
docs/source/reference_guides/hookspecs.md|
docs/source/tutorials/configuration.md|
docs/source/tutorials/debugging.md|
diff --git a/docs/source/how_to_guides/delayed_tasks.md b/docs/source/how_to_guides/delayed_tasks.md
index 16d0915e..16527a5d 100644
--- a/docs/source/how_to_guides/delayed_tasks.md
+++ b/docs/source/how_to_guides/delayed_tasks.md
@@ -15,9 +15,9 @@ files still as products of the task?
And how would you define a task that depends on these files. Or, how would define a
single task to process each file.
-The following sections will explain how delayed tasks work with pytask.
+The following sections will explain how you use pytask in these situations.
-## Delayed Products
+## Producing provisional nodes
Let us start with a task that downloads an unknown amount of files and stores them on
disk in a folder called `downloads`. As an example, we will download all files without a
@@ -29,16 +29,23 @@ emphasize-lines: 4, 11
---
```
-Since the names of the filesare not known when pytask is started, we need to use a
+Since the names of the files are not known when pytask is started, we need to use a
{class}`~pytask.DirectoryNode`. With a {class}`~pytask.DirectoryNode` we can specify
-where pytask can find the files and how they look like with an optional path and a glob
-pattern.
+where pytask can find the files. The files are described with a path (default is the
+directory of the task module) and a glob pattern (default is `*`).
When we use the {class}`~pytask.DirectoryNode` as a product annotation, we get access to
the `root_dir` as a {class}`~pathlib.Path` object inside the function which allows us to
store the files.
-## Delayed task
+:::{note}
+The {class}`~pytask.DirectoryNode` is a provisional node that implements
+{class}`~pytask.PProvisionalNode`. A provisional node is not a {class}`PNode`, but when
+its {meth}`~pytask.PProvisionalNode.collect` method is called, it returns actual nodes.
+A {class}`pytask.DirectoryNode`, for example, returns {class}`~pytask.PathNode`s.
+:::
+
+## Depending on provisional nodes
In the next step, we want to define a task that consumes all previously downloaded files
and merges them into one file.
@@ -49,16 +56,17 @@ emphasize-lines: 8-10
---
```
-When {class}`~pytask.DirectoryNode` is used as a dependency a list of all the files in
-the folder defined by the root path and the pattern are automatically collected and
-passed to the task.
+Here, we use a {class}`~pytask.DirectoryNode` as a dependency since we do not know the
+names of the downloaded files. Before the task is executed, the list of files in the
+folder defined by the root path and the pattern are automatically collected and passed
+to the task.
As long as we use a {class}`DirectoryNode` with the same `root_dir` and `pattern` in
both tasks, pytask will automatically recognize that the second task depends on the
first. If that is not true, you might need to make this dependency more explicit by
using {func}`@task(after=...) <pytask.task>` which is explained {ref}`here <after>`.
-## Delayed and repeated tasks
+## Task generators
Coming to the last use-case, what if we wanted to process each of the downloaded files
separately instead of dealing with them in one task?
From dd4ab173f7df6334f8324dcdf87092108df05f34 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 25 Nov 2023 13:17:54 +0100
Subject: [PATCH 56/81] Remove talking about delayed.
---
.pre-commit-config.yaml | 2 +-
docs/source/how_to_guides/index.md | 2 +-
..._tasks.md => provisional_nodes_and_task_generators.md} | 2 +-
src/_pytask/collect_utils.py | 8 ++++----
src/_pytask/delayed.py | 1 +
src/_pytask/execute.py | 2 +-
6 files changed, 9 insertions(+), 8 deletions(-)
rename docs/source/how_to_guides/{delayed_tasks.md => provisional_nodes_and_task_generators.md} (98%)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 8d6b781f..85062387 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -99,7 +99,7 @@ repos:
docs/source/how_to_guides/using_task_returns.md|
docs/source/how_to_guides/writing_custom_nodes.md|
docs/source/how_to_guides/hashing_inputs_of_tasks.md|
- docs/source/how_to_guides/delayed_tasks.md|
+ docs/source/how_to_guides/provisional_nodes_and_task_generators.md|
docs/source/reference_guides/hookspecs.md|
docs/source/tutorials/configuration.md|
docs/source/tutorials/debugging.md|
diff --git a/docs/source/how_to_guides/index.md b/docs/source/how_to_guides/index.md
index eb062e2d..4bab2a0c 100644
--- a/docs/source/how_to_guides/index.md
+++ b/docs/source/how_to_guides/index.md
@@ -17,7 +17,7 @@ capture_warnings
how_to_influence_build_order
hashing_inputs_of_tasks
using_task_returns
-delayed_tasks
+provisional_nodes_and_task_generators
writing_custom_nodes
how_to_write_a_plugin
the_data_catalog
diff --git a/docs/source/how_to_guides/delayed_tasks.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
similarity index 98%
rename from docs/source/how_to_guides/delayed_tasks.md
rename to docs/source/how_to_guides/provisional_nodes_and_task_generators.md
index 16527a5d..6e0c6bf8 100644
--- a/docs/source/how_to_guides/delayed_tasks.md
+++ b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -1,4 +1,4 @@
-# Delayed tasks
+# Provisional nodes and task generators
pytask's execution model can usually be separated into three phases.
diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py
index f18df266..be06efe1 100644
--- a/src/_pytask/collect_utils.py
+++ b/src/_pytask/collect_utils.py
@@ -293,7 +293,7 @@ def parse_dependencies_from_task_function(
if parameter_name in parameters_with_product_annot:
continue
- nodes = _collect_delayed_nodes_and_nodes(
+ nodes = _collect_nodes_and_provisional_nodes(
collect_dependency,
session,
node_path,
@@ -435,7 +435,7 @@ def parse_products_from_task_function( # noqa: C901
value = kwargs.get(parameter_name) or parameters_with_node_annot.get(
parameter_name
)
- collected_products = _collect_delayed_nodes_and_nodes(
+ collected_products = _collect_nodes_and_provisional_nodes(
_collect_product,
session,
node_path,
@@ -449,7 +449,7 @@ def parse_products_from_task_function( # noqa: C901
task_produces = obj.pytask_meta.produces if hasattr(obj, "pytask_meta") else None
if task_produces:
has_task_decorator = True
- collected_products = _collect_delayed_nodes_and_nodes(
+ collected_products = _collect_nodes_and_provisional_nodes(
_collect_product,
session,
node_path,
@@ -477,7 +477,7 @@ def parse_products_from_task_function( # noqa: C901
return out
-def _collect_delayed_nodes_and_nodes( # noqa: PLR0913
+def _collect_nodes_and_provisional_nodes( # noqa: PLR0913
collection_func: Callable[..., Any],
session: Session,
node_path: Path,
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 2015d38c..f8bf7ddb 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -1,3 +1,4 @@
+"""Contains hook implementations for provisional nodes and task generators."""
from __future__ import annotations
import inspect
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index b5099042..add901f8 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -154,7 +154,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C
)
raise NodeNotFoundError(msg)
- # Skip delayed nodes that are products since they do not have a state.
+ # Skip provisional nodes that are products since they do not have a state.
if node_signature not in predecessors and isinstance(
node, PProvisionalNode
):
From 432f78268fce677f24392e85911077201faf6ea2 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 28 Jan 2024 23:14:55 +0100
Subject: [PATCH 57/81] update
---
.../provisional_nodes_and_task_generators.md | 34 +++++++++----------
.../defining_dependencies_products.md | 2 --
src/_pytask/execute.py | 1 +
src/_pytask/pluginmanager.py | 1 +
tests/test_cache.py | 4 +++
tests/test_capture.py | 1 +
tests/test_collect.py | 4 +++
tests/test_collect_command.py | 1 +
tests/test_config.py | 3 ++
tests/test_dag.py | 1 +
tests/test_execute.py | 8 +++++
tests/test_hashlib.py | 1 +
tests/test_mark_structures.py | 2 ++
tests/test_nodes.py | 2 ++
tests/test_skipping.py | 2 ++
tests/test_task.py | 7 ++++
tests/test_traceback.py | 1 +
17 files changed, 56 insertions(+), 19 deletions(-)
diff --git a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
index 6e0c6bf8..ed27b4e0 100644
--- a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
+++ b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -6,11 +6,11 @@ pytask's execution model can usually be separated into three phases.
1. Building the DAG.
1. Executing the tasks.
-But, in some situations pytask needs to be more flexible.
+But, in some situations, pytask needs to be more flexible.
Imagine you want to download files from an online storage, but the total number of files
-and their filenames is unknown before the task has started. How can you describe the
-files still as products of the task?
+and their filenames is unknown before the task has started. How can you still describe
+the files as products of the task?
And how would you define a task that depends on these files. Or, how would define a
single task to process each file.
@@ -20,8 +20,8 @@ The following sections will explain how you use pytask in these situations.
## Producing provisional nodes
Let us start with a task that downloads an unknown amount of files and stores them on
-disk in a folder called `downloads`. As an example, we will download all files without a
-file extension from the pytask repository.
+disk in a folder called `downloads`. As an example, we will download all files from the
+pytask repository without a file extension.
```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_delayed_products.py
---
@@ -35,20 +35,20 @@ where pytask can find the files. The files are described with a path (default is
directory of the task module) and a glob pattern (default is `*`).
When we use the {class}`~pytask.DirectoryNode` as a product annotation, we get access to
-the `root_dir` as a {class}`~pathlib.Path` object inside the function which allows us to
-store the files.
+the `root_dir` as a {class}`~pathlib.Path` object inside the function, which allows us
+to store the files.
-:::{note}
+```{note}
The {class}`~pytask.DirectoryNode` is a provisional node that implements
-{class}`~pytask.PProvisionalNode`. A provisional node is not a {class}`PNode`, but when
+{class}`~pytask.PProvisionalNode`. A provisional node is not a {class}`~pytask.PNode`, but when
its {meth}`~pytask.PProvisionalNode.collect` method is called, it returns actual nodes.
-A {class}`pytask.DirectoryNode`, for example, returns {class}`~pytask.PathNode`s.
-:::
+A {class}`~pytask.DirectoryNode`, for example, returns {class}`~pytask.PathNode`.
+```
## Depending on provisional nodes
-In the next step, we want to define a task that consumes all previously downloaded files
-and merges them into one file.
+In the next step, we want to define a task that consumes and merges all previously
+downloaded files into one file.
```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_delayed_task.py
---
@@ -64,15 +64,15 @@ to the task.
As long as we use a {class}`DirectoryNode` with the same `root_dir` and `pattern` in
both tasks, pytask will automatically recognize that the second task depends on the
first. If that is not true, you might need to make this dependency more explicit by
-using {func}`@task(after=...) <pytask.task>` which is explained {ref}`here <after>`.
+using {func}`@task(after=...) <pytask.task>`, which is explained {ref}`here <after>`.
## Task generators
-Coming to the last use-case, what if we wanted to process each of the downloaded files
+Coming to the last use case, what if we wanted to process each of the downloaded files
separately instead of dealing with them in one task?
-To define an unknown amount of tasks for an unknown amount of downloaded files, we have
-to write a task generator.
+We have to write a task generator to define an unknown number of tasks for an unknown
+number of downloaded files.
A task generator is a task function in which we define more tasks, just as if we were
writing functions in a normal task module.
diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md
index 167fec39..71831554 100644
--- a/docs/source/tutorials/defining_dependencies_products.md
+++ b/docs/source/tutorials/defining_dependencies_products.md
@@ -243,8 +243,6 @@ You can do the same with dependencies.
(after)=
-(after)=
-
## Depending on a task
In some situations, you want to define a task depending on another task.
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 18a20a6d..cdf6d4ed 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -145,6 +145,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C
node = dag.nodes[node_signature].get("task") or dag.nodes[
node_signature
].get("node")
+
if node_signature in predecessors and not node.state():
msg = f"{task.name!r} requires missing node {node.name!r}."
if IS_FILE_SYSTEM_CASE_SENSITIVE:
diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py
index 931e12a8..27326eda 100644
--- a/src/_pytask/pluginmanager.py
+++ b/src/_pytask/pluginmanager.py
@@ -45,6 +45,7 @@ def pytask_add_hooks(pm: PluginManager) -> None:
"_pytask.dag_command",
"_pytask.database",
"_pytask.debugging",
+ "_pytask.delayed",
"_pytask.execute",
"_pytask.live",
"_pytask.logging",
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 74ac2a67..a6faf9b3 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -2,10 +2,12 @@
import inspect
+import pytest
from _pytask.cache import _make_memoize_key
from _pytask.cache import Cache
+@pytest.mark.unit()
def test_cache():
cache = Cache()
@@ -29,6 +31,7 @@ def func(a, b):
assert func.cache.cache_info.misses == 1
+@pytest.mark.unit()
def test_cache_add():
cache = Cache()
@@ -52,6 +55,7 @@ def func(a):
assert cache.cache_info.misses == 1
+@pytest.mark.unit()
def test_make_memoize_key():
def func(a, b): # pragma: no cover
return a + b
diff --git a/tests/test_capture.py b/tests/test_capture.py
index b78f93fe..0699fc81 100644
--- a/tests/test_capture.py
+++ b/tests/test_capture.py
@@ -112,6 +112,7 @@ def task_show_capture():
raise NotImplementedError
+@pytest.mark.end_to_end()
@pytest.mark.xfail(
sys.platform == "win32",
reason="from pytask ... cannot be found",
diff --git a/tests/test_collect.py b/tests/test_collect.py
index e9c928a8..3389c6cd 100644
--- a/tests/test_collect.py
+++ b/tests/test_collect.py
@@ -410,6 +410,7 @@ def task_example(path: Annotated[Path, Path("file.txt"), Product]) -> None: ...
assert "is defined twice" in result.output
+@pytest.mark.end_to_end()
@pytest.mark.parametrize(
"node",
[
@@ -431,6 +432,7 @@ def task_example(path = {node}): ...
assert all(i in result.output for i in ("only", "files", "are", "allowed"))
+@pytest.mark.end_to_end()
@pytest.mark.parametrize(
"node",
[
@@ -454,6 +456,7 @@ def task_example(path: Annotated[Any, Product] = {node}): ...
assert all(i in result.output for i in ("only", "files", "are", "allowed"))
+@pytest.mark.end_to_end()
@pytest.mark.parametrize(
"node",
[
@@ -479,6 +482,7 @@ def task_example() -> Annotated[str, {node}]:
assert session.tasks[0].produces["return"].name == tmp_path.name + "/file.txt"
+@pytest.mark.end_to_end()
def test_error_when_return_annotation_cannot_be_parsed(runner, tmp_path):
source = """
from typing_extensions import Annotated
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 139a92cf..aad3da69 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -672,6 +672,7 @@ def task_example({node_def}: ...
assert "/*.txt>" in captured
+@pytest.mark.end_to_end()
def test_collect_task_with_delayed_dependencies(runner, tmp_path):
source = """
from typing_extensions import Annotated
diff --git a/tests/test_config.py b/tests/test_config.py
index 2eb5300d..33698562 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -65,6 +65,7 @@ def test_passing_paths_via_configuration_file(tmp_path, file_or_folder):
assert len(session.tasks) == 1
+@pytest.mark.end_to_end()
def test_not_existing_path_in_config(runner, tmp_path):
config = """
[tool.pytask.ini_options]
@@ -76,6 +77,7 @@ def test_not_existing_path_in_config(runner, tmp_path):
assert result.exit_code == ExitCode.CONFIGURATION_FAILED
+@pytest.mark.end_to_end()
def test_paths_are_relative_to_configuration_file_cli(tmp_path):
tmp_path.joinpath("src").mkdir()
tmp_path.joinpath("tasks").mkdir()
@@ -96,6 +98,7 @@ def test_paths_are_relative_to_configuration_file_cli(tmp_path):
assert "1 Succeeded" in result.stdout.decode()
+@pytest.mark.end_to_end()
@pytest.mark.skipif(
sys.platform == "win32" and os.environ.get("CI") == "true",
reason="Windows does not pick up the right Python interpreter.",
diff --git a/tests/test_dag.py b/tests/test_dag.py
index f801b7cd..c638bd16 100644
--- a/tests/test_dag.py
+++ b/tests/test_dag.py
@@ -99,6 +99,7 @@ def task_example(produces = Path("file.txt")):
assert result.exit_code == ExitCode.OK
+@pytest.mark.end_to_end()
def test_python_nodes_are_unique(tmp_path):
tmp_path.joinpath("a").mkdir()
tmp_path.joinpath("a", "task_example.py").write_text("def task_example(a=1): pass")
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 435c14da..1f62410d 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -702,6 +702,7 @@ def task_example(
assert "_pytask/execute.py" not in result.output
+@pytest.mark.end_to_end()
def test_hashing_works(tmp_path):
"""Use subprocess or otherwise the cache is filled from other tests."""
source = """
@@ -726,6 +727,7 @@ def task_example() -> Annotated[str, Path("file.txt")]:
assert hashes == hashes_
+@pytest.mark.end_to_end()
def test_python_node_as_product_with_product_annotation(runner, tmp_path):
source = """
from typing_extensions import Annotated
@@ -746,6 +748,7 @@ def task_write_file(text: Annotated[str, node]) -> Annotated[str, Path("file.txt
assert tmp_path.joinpath("file.txt").read_text() == "Hello, World!"
+@pytest.mark.end_to_end()
def test_pickle_node_as_product_with_product_annotation(runner, tmp_path):
source = """
from typing_extensions import Annotated
@@ -822,6 +825,7 @@ def task_d(path=Path("../bld/in.txt"), produces=Path("out.txt")):
assert "bld/in.txt" in result.output
+@pytest.mark.end_to_end()
def test_error_when_node_state_throws_error(runner, tmp_path):
source = """
from pytask import PythonNode
@@ -836,6 +840,7 @@ def task_example(a = PythonNode(value={"a": 1}, hash=True)):
assert "TypeError: unhashable type: 'dict'" in result.output
+@pytest.mark.end_to_end()
def test_task_is_not_reexecuted(runner, tmp_path):
source = """
from typing_extensions import Annotated
@@ -860,6 +865,7 @@ def task_second(path = Path("out.txt")) -> Annotated[str, Path("copy.txt")]:
assert "1 Skipped because unchanged" in result.output
+@pytest.mark.end_to_end()
def test_use_functional_interface_with_task(tmp_path):
def func(path):
path.touch()
@@ -878,6 +884,7 @@ def func(path):
assert session.exit_code == ExitCode.OK
+@pytest.mark.end_to_end()
def test_collect_task(runner, tmp_path):
source = """
from pytask import Task, PathNode
@@ -898,6 +905,7 @@ def func(path): path.touch()
assert tmp_path.joinpath("out.txt").exists()
+@pytest.mark.end_to_end()
def test_collect_task_without_path(runner, tmp_path):
source = """
from pytask import TaskWithoutPath, PathNode
diff --git a/tests/test_hashlib.py b/tests/test_hashlib.py
index a25c7949..6fec4f1e 100644
--- a/tests/test_hashlib.py
+++ b/tests/test_hashlib.py
@@ -6,6 +6,7 @@
from _pytask._hashlib import hash_value
+@pytest.mark.unit()
@pytest.mark.parametrize(
("value", "expected"),
[
diff --git a/tests/test_mark_structures.py b/tests/test_mark_structures.py
index 30540902..f0b554f9 100644
--- a/tests/test_mark_structures.py
+++ b/tests/test_mark_structures.py
@@ -4,6 +4,7 @@
import pytest
+@pytest.mark.unit()
@pytest.mark.parametrize(
("lhs", "rhs", "expected"),
[
@@ -17,6 +18,7 @@ def test__eq__(lhs, rhs, expected) -> None:
assert (lhs == rhs) == expected
+@pytest.mark.unit()
@pytest.mark.filterwarnings("ignore:Unknown pytask\\.mark\\.foo")
def test_aliases() -> None:
md = pytask.mark.foo(1, "2", three=3)
diff --git a/tests/test_nodes.py b/tests/test_nodes.py
index b396e326..48bda1a6 100644
--- a/tests/test_nodes.py
+++ b/tests/test_nodes.py
@@ -31,6 +31,7 @@ def test_hash_of_python_node(value, hash_, expected):
assert state == expected
+@pytest.mark.unit()
@pytest.mark.parametrize(
("node", "expected"),
[
@@ -110,6 +111,7 @@ def test_hash_of_pickle_node(tmp_path, value, exists, expected):
assert state is expected
+@pytest.mark.unit()
@pytest.mark.parametrize(
("node", "protocol", "expected"),
[
diff --git a/tests/test_skipping.py b/tests/test_skipping.py
index fbe0606d..981a1f1e 100644
--- a/tests/test_skipping.py
+++ b/tests/test_skipping.py
@@ -260,6 +260,7 @@ def test_pytask_execute_task_setup(marker_name, force, expectation):
pytask_execute_task_setup(session=session, task=task)
+@pytest.mark.end_to_end()
def test_skip_has_precendence_over_ancestor_failed(runner, tmp_path):
source = """
from pathlib import Path
@@ -276,6 +277,7 @@ def task_example_2(path=Path("file.txt")): ...
assert "1 Skipped" in result.output
+@pytest.mark.end_to_end()
def test_skipif_has_precendence_over_ancestor_failed(runner, tmp_path):
source = """
from pathlib import Path
diff --git a/tests/test_task.py b/tests/test_task.py
index fdaeee5c..a66e3573 100644
--- a/tests/test_task.py
+++ b/tests/test_task.py
@@ -570,6 +570,7 @@ def task_example():
assert tmp_path.joinpath("file2.txt").read_text() == "World!"
+@pytest.mark.end_to_end()
def test_error_when_function_is_defined_outside_loop_body(runner, tmp_path):
source = """
from pathlib import Path
@@ -589,6 +590,7 @@ def func(path: Annotated[Path, Product]):
assert "id=None" in result.output
+@pytest.mark.end_to_end()
def test_error_when_function_is_defined_outside_loop_body_with_id(runner, tmp_path):
source = """
from pathlib import Path
@@ -609,6 +611,7 @@ def func(path: Annotated[Path, Product]):
assert "id=b.txt" in result.output
+@pytest.mark.end_to_end()
def test_task_will_be_executed_after_another_one_with_string(runner, tmp_path):
source = """
from pytask import task
@@ -637,6 +640,7 @@ def task_first() -> Annotated[str, Path("out.txt")]:
assert "1 Skipped because unchanged" in result.output
+@pytest.mark.end_to_end()
def test_task_will_be_executed_after_another_one_with_function(tmp_path):
source = """
from pytask import task
@@ -656,6 +660,7 @@ def task_second():
assert session.exit_code == ExitCode.OK
+@pytest.mark.end_to_end()
def test_raise_error_for_wrong_after_expression(runner, tmp_path):
source = """
from pytask import task
@@ -673,6 +678,7 @@ def task_example() -> Annotated[str, Path("out.txt")]:
assert "Wrong expression passed to 'after'" in result.output
+@pytest.mark.end_to_end()
def test_raise_error_with_builtin_function_as_task(runner, tmp_path):
source = """
from pytask import task
@@ -690,6 +696,7 @@ def test_raise_error_with_builtin_function_as_task(runner, tmp_path):
assert "Builtin functions cannot be wrapped" in result.output
+@pytest.mark.end_to_end()
def test_task_function_in_another_module(runner, tmp_path):
source = """
def func():
diff --git a/tests/test_traceback.py b/tests/test_traceback.py
index c626d0c7..6ffaefa5 100644
--- a/tests/test_traceback.py
+++ b/tests/test_traceback.py
@@ -45,6 +45,7 @@ def helper():
assert ("This variable should not be shown." in result.output) is not is_hidden
+@pytest.mark.unit()
def test_render_traceback_with_string_traceback():
traceback = Traceback((Exception, Exception("Help"), "String traceback."))
rendered = render_to_string(traceback, console)
From fe640209664f343f99f3754d04d7ffe3950ad012 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 28 Jan 2024 23:55:40 +0100
Subject: [PATCH 58/81] Fix.
---
.../how_to_guides/provisional_nodes_and_task_generators.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
index ed27b4e0..1423d4d0 100644
--- a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
+++ b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -52,7 +52,7 @@ downloaded files into one file.
```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_delayed_task.py
---
-emphasize-lines: 8-10
+emphasize-lines: 9
---
```
From e7ca69d7ad5e3145aef73911c4dedbf8a03c31e7 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Mon, 29 Jan 2024 13:22:41 +0100
Subject: [PATCH 59/81] Fix.
---
tests/conftest.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/tests/conftest.py b/tests/conftest.py
index 673d0b04..a2380752 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import os
import re
import sys
from contextlib import contextmanager
@@ -112,7 +111,6 @@ def runner():
def pytest_collection_modifyitems(session, config, items) -> None: # noqa: ARG001
"""Add markers to Jupyter notebook tests."""
- if sys.platform == "darwin" and "CI" in os.environ: # pragma: no cover
- for item in items:
- if isinstance(item, NotebookItem):
- item.add_marker(pytest.mark.xfail(reason="Fails regularly on MacOS"))
+ for item in items:
+ if isinstance(item, NotebookItem):
+ item.add_marker(pytest.mark.xfail(reason="The tests are flaky."))
From 5b4cba2a1fb8f916f09193f202e7710c69711337 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Tue, 30 Jan 2024 23:08:07 +0100
Subject: [PATCH 60/81] Rename generator to is_generator.
---
.../how_to_guides/delayed_tasks_task_generator.py | 2 +-
src/_pytask/task_utils.py | 8 ++++----
tests/test_execute.py | 12 ++++++------
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/docs_src/how_to_guides/delayed_tasks_task_generator.py b/docs_src/how_to_guides/delayed_tasks_task_generator.py
index e2390d4e..435643a7 100644
--- a/docs_src/how_to_guides/delayed_tasks_task_generator.py
+++ b/docs_src/how_to_guides/delayed_tasks_task_generator.py
@@ -5,7 +5,7 @@
from typing_extensions import Annotated
-@task(generator=True)
+@task(is_generator=True)
def task_copy_files(
paths: Annotated[
list[Path], DirectoryNode(root_dir=Path("downloads"), pattern="*")
diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py
index 21c0cffb..4496914f 100644
--- a/src/_pytask/task_utils.py
+++ b/src/_pytask/task_utils.py
@@ -42,7 +42,7 @@ def task( # noqa: PLR0913
name: str | None = None,
*,
after: str | Callable[..., Any] | list[Callable[..., Any]] | None = None,
- generator: bool = False,
+ is_generator: bool = False,
id: str | None = None, # noqa: A002
kwargs: dict[Any, Any] | None = None,
produces: Any | None = None,
@@ -63,7 +63,7 @@ def task( # noqa: PLR0913
An expression or a task function or a list of task functions that need to be
executed before this task can be executed. See :ref:`after` for more
information.
- generator
+ is_generator
An indicator whether this task is a task generator.
id
An id for the task if it is part of a parametrization. Otherwise, an automatic
@@ -135,7 +135,7 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
if hasattr(unwrapped, "pytask_meta"):
unwrapped.pytask_meta.after = parsed_after
- unwrapped.pytask_meta.is_generator = generator
+ unwrapped.pytask_meta.is_generator = is_generator
unwrapped.pytask_meta.id_ = id
unwrapped.pytask_meta.kwargs = parsed_kwargs
unwrapped.pytask_meta.markers.append(Mark("task", (), {}))
@@ -145,7 +145,7 @@ def wrapper(func: Callable[..., Any]) -> Callable[..., Any]:
else:
unwrapped.pytask_meta = CollectionMetadata(
after=parsed_after,
- is_generator=generator,
+ is_generator=is_generator,
id_=id,
kwargs=parsed_kwargs,
markers=[Mark("task", (), {})],
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 1f62410d..47a036cc 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1073,7 +1073,7 @@ def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]:
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
- @task(after=task_produces, generator=True)
+ @task(after=task_produces, is_generator=True)
def task_depends(
paths = DirectoryNode(pattern="[ab].txt")
) -> ...:
@@ -1109,7 +1109,7 @@ def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]:
path.joinpath("a.txt").write_text("Hello, ")
path.joinpath("b.txt").write_text("World!")
- @task(after=task_produces, generator=True)
+ @task(after=task_produces, is_generator=True)
def task_depends(
paths = DirectoryNode(pattern="[ab].txt")
) -> ...:
@@ -1146,7 +1146,7 @@ def task_produces() -> Annotated[None, DirectoryNode(pattern="[a].txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
- @task(after=task_produces, generator=True)
+ @task(after=task_produces, is_generator=True)
def task_depends(
paths = DirectoryNode(pattern="[a].txt")
) -> ...:
@@ -1180,7 +1180,7 @@ def task_produces() -> Annotated[None, DirectoryNode(pattern="[a].txt")]:
path = Path(__file__).parent
path.joinpath("a.txt").write_text("Hello, ")
- @task(after=task_produces, generator=True)
+ @task(after=task_produces, is_generator=True)
def task_depends(
paths = DirectoryNode(pattern="[a].txt")
) -> ...:
@@ -1212,7 +1212,7 @@ def test_gracefully_fail_when_task_generator_raises_error(runner, tmp_path):
from pytask import DirectoryNode, task, Product
from pathlib import Path
- @task(generator=True)
+ @task(is_generator=True)
def task_example(
root_dir: Annotated[Path, DirectoryNode(pattern="[a].txt"), Product]
) -> ...:
@@ -1233,7 +1233,7 @@ def test_use_delayed_node_as_product_in_generator_without_rerun(runner, tmp_path
from pytask import DirectoryNode, task, Product
from pathlib import Path
- @task(generator=True)
+ @task(is_generator=True)
def task_example(
root_dir: Annotated[Path, DirectoryNode(pattern="[ab].txt"), Product]
) -> ...:
From deb45d1d6f5eec345c3e46b552f954294ee261dd Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 01:12:04 +0100
Subject: [PATCH 61/81] Fix side-effect in tests.
---
tests/test_task_utils.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py
index 21c2acea..47bb14a5 100644
--- a/tests/test_task_utils.py
+++ b/tests/test_task_utils.py
@@ -2,12 +2,14 @@
from contextlib import ExitStack as does_not_raise # noqa: N813
from functools import partial
+from pathlib import Path
from typing import NamedTuple
import pytest
from _pytask.task_utils import _arg_value_to_id_component
from _pytask.task_utils import _parse_name
from _pytask.task_utils import _parse_task_kwargs
+from _pytask.task_utils import COLLECTED_TASKS
from attrs import define
from pytask import Mark
from pytask import task
@@ -76,6 +78,9 @@ def task_example():
assert task_example.pytask_meta.name == "task_example"
assert task_example.pytask_meta.produces is None
+ # Remove collected task.
+ COLLECTED_TASKS.pop(Path(__file__))
+
def task_func(x): # noqa: ARG001 # pragma: no cover
pass
From 86f227d2fec35d229c3be2bbbec1100363e0d38a Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
<66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 4 Feb 2024 00:12:58 +0000
Subject: [PATCH 62/81] [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
---
docs/source/reference_guides/api.md | 6 ++----
.../tutorials/repeating_tasks_with_different_inputs.md | 6 ++----
docs/source/tutorials/selecting_tasks.md | 3 +--
docs/source/tutorials/skipping_tasks.md | 5 +++--
docs/source/tutorials/write_a_task.md | 6 ++----
5 files changed, 10 insertions(+), 16 deletions(-)
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index d25f7953..22872173 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,8 +136,7 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function():
- ...
+def task_function(): ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -154,8 +153,7 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function():
- ...
+def task_function(): ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index 8e5a10e5..b369167b 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,8 +268,7 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces):
- ...
+ def task_create_random_data(seed, produces): ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -289,8 +288,7 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces):
- ...
+ def task_create_random_data(i, produces): ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 266b47bf..23b06903 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,8 +91,7 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i):
- ...
+ def task_parametrized(i=i): ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index 7278ee3e..e223b1cd 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -42,8 +42,9 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
-def task_that_takes_really_long_to_run(path: Path = Path("time_intensive_product.pkl")):
- ...
+def task_that_takes_really_long_to_run(
+ path: Path = Path("time_intensive_product.pkl"),
+): ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9d949077..9ecc3bbc 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,16 +117,14 @@ from pytask import task
@task
-def create_random_data():
- ...
+def create_random_data(): ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data():
- ...
+def create_random_data(): ...
```
## Customize task module names
From 11e34845460f7058c963a95a58ae0236980e78ad Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 01:16:37 +0100
Subject: [PATCH 63/81] Fix.
---
docs/source/reference_guides/api.md | 6 ++++--
.../tutorials/repeating_tasks_with_different_inputs.md | 6 ++++--
docs/source/tutorials/selecting_tasks.md | 3 ++-
docs/source/tutorials/skipping_tasks.md | 3 ++-
docs/source/tutorials/write_a_task.md | 6 ++++--
5 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index 22872173..d25f7953 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,7 +136,8 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function(): ...
+def task_function():
+ ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -153,7 +154,8 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function(): ...
+def task_function():
+ ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index b369167b..8e5a10e5 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,7 +268,8 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces): ...
+ def task_create_random_data(seed, produces):
+ ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -288,7 +289,8 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces): ...
+ def task_create_random_data(i, produces):
+ ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 23b06903..266b47bf 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,7 +91,8 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i): ...
+ def task_parametrized(i=i):
+ ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index e223b1cd..23857f6a 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -44,7 +44,8 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
def task_that_takes_really_long_to_run(
path: Path = Path("time_intensive_product.pkl"),
-): ...
+):
+ ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9ecc3bbc..9d949077 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,14 +117,16 @@ from pytask import task
@task
-def create_random_data(): ...
+def create_random_data():
+ ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data(): ...
+def create_random_data():
+ ...
```
## Customize task module names
From ab47e5d897147bccf9e2663ee1e3a50014dc01cd Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 01:17:21 +0100
Subject: [PATCH 64/81] Fix.
---
.../how_to_guides/provisional_nodes_and_task_generators.md | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
index 1423d4d0..92ebb178 100644
--- a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
+++ b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -40,9 +40,10 @@ to store the files.
```{note}
The {class}`~pytask.DirectoryNode` is a provisional node that implements
-{class}`~pytask.PProvisionalNode`. A provisional node is not a {class}`~pytask.PNode`, but when
-its {meth}`~pytask.PProvisionalNode.collect` method is called, it returns actual nodes.
-A {class}`~pytask.DirectoryNode`, for example, returns {class}`~pytask.PathNode`.
+{class}`~pytask.PProvisionalNode`. A provisional node is not a {class}`~pytask.PNode`,
+but when its {meth}`~pytask.PProvisionalNode.collect` method is called, it returns
+actual nodes. A {class}`~pytask.DirectoryNode`, for example, returns
+{class}`~pytask.PathNode`.
```
## Depending on provisional nodes
From b683da51efdb18294c780f4f851f026a039ea71e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
<66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 4 Feb 2024 00:17:57 +0000
Subject: [PATCH 65/81] [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
---
docs/source/reference_guides/api.md | 6 ++----
.../tutorials/repeating_tasks_with_different_inputs.md | 6 ++----
docs/source/tutorials/selecting_tasks.md | 3 +--
docs/source/tutorials/skipping_tasks.md | 3 +--
docs/source/tutorials/write_a_task.md | 6 ++----
5 files changed, 8 insertions(+), 16 deletions(-)
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index d25f7953..22872173 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,8 +136,7 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function():
- ...
+def task_function(): ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -154,8 +153,7 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function():
- ...
+def task_function(): ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index 8e5a10e5..b369167b 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,8 +268,7 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces):
- ...
+ def task_create_random_data(seed, produces): ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -289,8 +288,7 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces):
- ...
+ def task_create_random_data(i, produces): ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 266b47bf..23b06903 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,8 +91,7 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i):
- ...
+ def task_parametrized(i=i): ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index 23857f6a..e223b1cd 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -44,8 +44,7 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
def task_that_takes_really_long_to_run(
path: Path = Path("time_intensive_product.pkl"),
-):
- ...
+): ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9d949077..9ecc3bbc 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,16 +117,14 @@ from pytask import task
@task
-def create_random_data():
- ...
+def create_random_data(): ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data():
- ...
+def create_random_data(): ...
```
## Customize task module names
From f56fa689b05974daddc0a96f1f501010ac624bec Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 01:40:36 +0100
Subject: [PATCH 66/81] Fix docs and coverage.
---
.../provisional_nodes_and_task_generators.md | 32 +++++++++----------
src/_pytask/node_protocols.py | 2 +-
src/_pytask/nodes.py | 3 +-
3 files changed, 18 insertions(+), 19 deletions(-)
diff --git a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
index 92ebb178..c63a8894 100644
--- a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
+++ b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -12,16 +12,14 @@ Imagine you want to download files from an online storage, but the total number
and their filenames is unknown before the task has started. How can you still describe
the files as products of the task?
-And how would you define a task that depends on these files. Or, how would define a
-single task to process each file.
+And how would you define another task that depends on these files?
The following sections will explain how you use pytask in these situations.
## Producing provisional nodes
-Let us start with a task that downloads an unknown amount of files and stores them on
-disk in a folder called `downloads`. As an example, we will download all files from the
-pytask repository without a file extension.
+Let us start with a task that downloads all files without an extension from the root
+folder of the pytask repository and stores them on disk in a folder called `downloads`.
```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_delayed_products.py
---
@@ -57,29 +55,29 @@ emphasize-lines: 9
---
```
-Here, we use a {class}`~pytask.DirectoryNode` as a dependency since we do not know the
+Here, we use the {class}`~pytask.DirectoryNode` as a dependency since we do not know the
names of the downloaded files. Before the task is executed, the list of files in the
folder defined by the root path and the pattern are automatically collected and passed
to the task.
-As long as we use a {class}`DirectoryNode` with the same `root_dir` and `pattern` in
-both tasks, pytask will automatically recognize that the second task depends on the
-first. If that is not true, you might need to make this dependency more explicit by
-using {func}`@task(after=...) <pytask.task>`, which is explained {ref}`here <after>`.
+If we use a {class}`DirectoryNode` with the same `root_dir` and `pattern` in both tasks,
+pytask will automatically recognize that the second task depends on the first. If that
+is not true, you might need to make this dependency more explicit by using
+{func}`@task(after=...) <pytask.task>`, which is explained {ref}`here <after>`.
## Task generators
-Coming to the last use case, what if we wanted to process each of the downloaded files
-separately instead of dealing with them in one task?
+What if we wanted to process each downloaded file separately instead of dealing with
+them in one task?
-We have to write a task generator to define an unknown number of tasks for an unknown
-number of downloaded files.
+For that, we have to write a task generator to define an unknown number of tasks for an
+unknown number of downloaded files.
A task generator is a task function in which we define more tasks, just as if we were
-writing functions in a normal task module.
+writing functions in a task module.
-As an example, each task takes one of the downloaded files and copies its content to a
-`.txt` file.
+The code snippet shows each task takes one of the downloaded files and copies its
+content to a `.txt` file.
```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_task_generator.py
```
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index 034d3b9c..7b923724 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -121,7 +121,7 @@ class PProvisionalNode(Protocol):
def signature(self) -> str:
"""Return the signature of the node."""
- def load(self, is_product: bool = False) -> Any:
+ def load(self, is_product: bool = False) -> Any: # pragma: no cover
"""Load a probisional node.
A provisional node will never be loaded as a dependency since it would be
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index e8c06c03..a9485d5a 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -381,7 +381,8 @@ def signature(self) -> str:
def load(self, is_product: bool = False) -> Path:
if is_product:
return self.root_dir # type: ignore[return-value]
- raise NotImplementedError
+ msg = "'DirectoryNode' cannot be loaded as a dependency"
+ raise NotImplementedError(msg) # pragma: no cover
def collect(self) -> list[Path]:
"""Collect paths defined by the pattern."""
From 240c47e772780495b44f90cfb1161eb74eab6bd2 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 11:35:56 +0100
Subject: [PATCH 67/81] Add test for task generator in notebook.
---
docs/source/reference_guides/api.md | 6 +-
.../repeating_tasks_with_different_inputs.md | 6 +-
docs/source/tutorials/selecting_tasks.md | 3 +-
docs/source/tutorials/skipping_tasks.md | 3 +-
docs/source/tutorials/write_a_task.md | 6 +-
src/_pytask/delayed.py | 5 +-
tests/test_execute.py | 2 +-
tests/test_jupyter/test_task_generator.ipynb | 80 +++++++++++++++++++
8 files changed, 101 insertions(+), 10 deletions(-)
create mode 100644 tests/test_jupyter/test_task_generator.ipynb
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index 22872173..d25f7953 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,7 +136,8 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function(): ...
+def task_function():
+ ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -153,7 +154,8 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function(): ...
+def task_function():
+ ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index b369167b..8e5a10e5 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,7 +268,8 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces): ...
+ def task_create_random_data(seed, produces):
+ ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -288,7 +289,8 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces): ...
+ def task_create_random_data(i, produces):
+ ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 23b06903..266b47bf 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,7 +91,8 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i): ...
+ def task_parametrized(i=i):
+ ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index e223b1cd..23857f6a 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -44,7 +44,8 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
def task_that_takes_really_long_to_run(
path: Path = Path("time_intensive_product.pkl"),
-): ...
+):
+ ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9ecc3bbc..9d949077 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,14 +117,16 @@ from pytask import task
@task
-def create_random_data(): ...
+def create_random_data():
+ ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data(): ...
+def create_random_data():
+ ...
```
## Customize task module names
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index f8bf7ddb..77b74676 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -51,7 +51,7 @@ def _safe_load(node: PNode | PProvisionalNode, task: PTask, is_product: bool) ->
_ERROR_TASK_GENERATOR_RETURN = """\
Could not collect return of task generator. The return should be a task function or a \
-task class but received {obj} instead."""
+task class but received {} instead."""
@hookimpl
@@ -76,6 +76,9 @@ def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, P
if isinstance(task, PTaskWithPath) and task.path in COLLECTED_TASKS:
tasks = COLLECTED_TASKS.pop(task.path)
name_to_function = parse_collected_tasks_with_task_marker(tasks)
+ elif isinstance(task, PTask) and None in COLLECTED_TASKS:
+ tasks = COLLECTED_TASKS.pop(None)
+ name_to_function = parse_collected_tasks_with_task_marker(tasks)
else:
# Parse individual tasks.
if is_task_function(out) or isinstance(out, PTask):
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 47a036cc..a00aefdb 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1076,7 +1076,7 @@ def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]:
@task(after=task_produces, is_generator=True)
def task_depends(
paths = DirectoryNode(pattern="[ab].txt")
- ) -> ...:
+ ):
for path in paths:
@task
diff --git a/tests/test_jupyter/test_task_generator.ipynb b/tests/test_jupyter/test_task_generator.ipynb
new file mode 100644
index 00000000..703352a5
--- /dev/null
+++ b/tests/test_jupyter/test_task_generator.ipynb
@@ -0,0 +1,80 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "12bc75b1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from __future__ import annotations\n",
+ "\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from typing_extensions import Annotated\n",
+ "\n",
+ "import pytask\n",
+ "from pytask import DirectoryNode, ExitCode, task"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "29ac7311",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "@task()\n",
+ "def task_create_files() -> Annotated[None, DirectoryNode(pattern=\"[ab].txt\")]:\n",
+ " path = Path()\n",
+ " path.joinpath(\"a.txt\").write_text(\"Hello, \")\n",
+ " path.joinpath(\"b.txt\").write_text(\"World!\")\n",
+ "\n",
+ "\n",
+ "@task(after=task_create_files, is_generator=True)\n",
+ "def task_generator_copy_files(\n",
+ " paths: Annotated[list[Path], DirectoryNode(pattern=\"[ab].txt\")]\n",
+ "):\n",
+ " for path in paths:\n",
+ "\n",
+ " @task\n",
+ " def task_copy(\n",
+ " path: Path = path,\n",
+ " ) -> Annotated[str, path.with_name(path.stem + \"-copy.txt\")]:\n",
+ " return path.read_text()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "738c9418",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "session = pytask.build(tasks=[task_create_files, task_generator_copy_files])\n",
+ "assert session.exit_code == ExitCode.OK"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
From 7d8829fbd4c0fd15614faecc67883b4f1e9771c3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
<66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 4 Feb 2024 10:40:04 +0000
Subject: [PATCH 68/81] [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
---
docs/source/reference_guides/api.md | 6 ++----
.../tutorials/repeating_tasks_with_different_inputs.md | 6 ++----
docs/source/tutorials/selecting_tasks.md | 3 +--
docs/source/tutorials/skipping_tasks.md | 3 +--
docs/source/tutorials/write_a_task.md | 6 ++----
5 files changed, 8 insertions(+), 16 deletions(-)
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index d25f7953..22872173 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,8 +136,7 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function():
- ...
+def task_function(): ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -154,8 +153,7 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function():
- ...
+def task_function(): ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index 8e5a10e5..b369167b 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,8 +268,7 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces):
- ...
+ def task_create_random_data(seed, produces): ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -289,8 +288,7 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces):
- ...
+ def task_create_random_data(i, produces): ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 266b47bf..23b06903 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,8 +91,7 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i):
- ...
+ def task_parametrized(i=i): ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index 23857f6a..e223b1cd 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -44,8 +44,7 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
def task_that_takes_really_long_to_run(
path: Path = Path("time_intensive_product.pkl"),
-):
- ...
+): ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9d949077..9ecc3bbc 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,16 +117,14 @@ from pytask import task
@task
-def create_random_data():
- ...
+def create_random_data(): ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data():
- ...
+def create_random_data(): ...
```
## Customize task module names
From 01a00c2941ae574eac4c2eae3359ad6c2e9a124c Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 14:41:48 +0100
Subject: [PATCH 69/81] FIx.
---
docs/source/reference_guides/api.md | 6 ++++--
.../tutorials/repeating_tasks_with_different_inputs.md | 6 ++++--
docs/source/tutorials/selecting_tasks.md | 3 ++-
docs/source/tutorials/skipping_tasks.md | 3 ++-
docs/source/tutorials/write_a_task.md | 6 ++++--
5 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index 22872173..d25f7953 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,7 +136,8 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function(): ...
+def task_function():
+ ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -153,7 +154,8 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function(): ...
+def task_function():
+ ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index b369167b..8e5a10e5 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,7 +268,8 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces): ...
+ def task_create_random_data(seed, produces):
+ ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -288,7 +289,8 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces): ...
+ def task_create_random_data(i, produces):
+ ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 23b06903..266b47bf 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,7 +91,8 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i): ...
+ def task_parametrized(i=i):
+ ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index 13194d4c..6e511a5b 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -44,7 +44,8 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
def task_that_takes_really_long_to_run(
path: Path = Path("time_intensive_product.pkl"),
-): ...
+):
+ ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9ecc3bbc..9d949077 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,14 +117,16 @@ from pytask import task
@task
-def create_random_data(): ...
+def create_random_data():
+ ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data(): ...
+def create_random_data():
+ ...
```
## Customize task module names
From 0bff197410cc5d3c24ba84d39fd7af35874e6af5 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
<66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 4 Feb 2024 13:42:29 +0000
Subject: [PATCH 70/81] [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
---
docs/source/reference_guides/api.md | 6 ++----
.../tutorials/repeating_tasks_with_different_inputs.md | 6 ++----
docs/source/tutorials/selecting_tasks.md | 3 +--
docs/source/tutorials/skipping_tasks.md | 3 +--
docs/source/tutorials/write_a_task.md | 6 ++----
5 files changed, 8 insertions(+), 16 deletions(-)
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index d25f7953..22872173 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,8 +136,7 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function():
- ...
+def task_function(): ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -154,8 +153,7 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function():
- ...
+def task_function(): ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index 8e5a10e5..b369167b 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,8 +268,7 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces):
- ...
+ def task_create_random_data(seed, produces): ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -289,8 +288,7 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces):
- ...
+ def task_create_random_data(i, produces): ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 266b47bf..23b06903 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,8 +91,7 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i):
- ...
+ def task_parametrized(i=i): ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index 6e511a5b..13194d4c 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -44,8 +44,7 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
def task_that_takes_really_long_to_run(
path: Path = Path("time_intensive_product.pkl"),
-):
- ...
+): ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9d949077..9ecc3bbc 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,16 +117,14 @@ from pytask import task
@task
-def create_random_data():
- ...
+def create_random_data(): ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data():
- ...
+def create_random_data(): ...
```
## Customize task module names
From c3b6122f2c73da6fb2f1c10f0548cd61f40c611d Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 17:25:33 +0100
Subject: [PATCH 71/81] Fix.
---
src/_pytask/delayed.py | 2 +-
tests/test_execute.py | 60 ++++++++------------
tests/test_jupyter/test_task_generator.ipynb | 1 -
3 files changed, 26 insertions(+), 37 deletions(-)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 77b74676..8eb6be63 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -76,7 +76,7 @@ def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, P
if isinstance(task, PTaskWithPath) and task.path in COLLECTED_TASKS:
tasks = COLLECTED_TASKS.pop(task.path)
name_to_function = parse_collected_tasks_with_task_marker(tasks)
- elif isinstance(task, PTask) and None in COLLECTED_TASKS:
+ elif None in COLLECTED_TASKS:
tasks = COLLECTED_TASKS.pop(None)
name_to_function = parse_collected_tasks_with_task_marker(tasks)
else:
diff --git a/tests/test_execute.py b/tests/test_execute.py
index a00aefdb..80e85f96 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1004,7 +1004,7 @@ def task_example(
@pytest.mark.end_to_end()
-def test_task_that_depends_on_delayed_task(tmp_path):
+def test_task_that_depends_on_delayed_task(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, task
@@ -1024,12 +1024,10 @@ def task_depends(
"""
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
- session = build(paths=tmp_path)
-
- assert session.exit_code == ExitCode.OK
- assert len(session.tasks) == 2
- assert len(session.tasks[0].produces["return"]) == 2
- assert len(session.tasks[1].depends_on["paths"]) == 2
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "2 Collected tasks" in result.output
+ assert "2 Succeeded" in result.output
@pytest.mark.end_to_end()
@@ -1062,7 +1060,7 @@ def task_depends(
@pytest.mark.end_to_end()
-def test_delayed_task_generation(tmp_path):
+def test_delayed_task_generation(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, task
@@ -1087,18 +1085,16 @@ def task_copy(
"""
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
- session = build(paths=tmp_path)
-
- assert session.exit_code == ExitCode.OK
- assert len(session.tasks) == 4
- assert len(session.tasks[0].produces["return"]) == 2
- assert len(session.tasks[1].depends_on["paths"]) == 2
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "4 Collected tasks" in result.output
+ assert "4 Succeeded" in result.output
assert tmp_path.joinpath("a-copy.txt").exists()
assert tmp_path.joinpath("b-copy.txt").exists()
@pytest.mark.end_to_end()
-def test_delayed_task_generation_with_generator(tmp_path):
+def test_delayed_task_generation_with_generator(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, task
@@ -1125,18 +1121,16 @@ def task_copy(
"""
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
- session = build(paths=tmp_path)
-
- assert session.exit_code == ExitCode.OK
- assert len(session.tasks) == 4
- assert len(session.tasks[0].produces["return"]) == 2
- assert len(session.tasks[1].depends_on["paths"]) == 2
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "4 Collected tasks" in result.output
+ assert "4 Succeeded" in result.output
assert tmp_path.joinpath("a-copy.txt").exists()
assert tmp_path.joinpath("b-copy.txt").exists()
@pytest.mark.end_to_end()
-def test_delayed_task_generation_with_single_function(tmp_path):
+def test_delayed_task_generation_with_single_function(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, task
@@ -1160,17 +1154,15 @@ def task_copy(
"""
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
- session = build(paths=tmp_path)
-
- assert session.exit_code == ExitCode.OK
- assert len(session.tasks) == 3
- assert len(session.tasks[0].produces["return"]) == 1
- assert len(session.tasks[1].depends_on["paths"]) == 1
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "3 Collected tasks" in result.output
+ assert "3 Succeeded" in result.output
assert tmp_path.joinpath("a-copy.txt").exists()
@pytest.mark.end_to_end()
-def test_delayed_task_generation_with_task_node(tmp_path):
+def test_delayed_task_generation_with_task_node(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, TaskWithoutPath, task, PathNode
@@ -1196,12 +1188,10 @@ def task_depends(
"""
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
- session = build(paths=tmp_path)
-
- assert session.exit_code == ExitCode.OK
- assert len(session.tasks) == 3
- assert len(session.tasks[0].produces["return"]) == 1
- assert len(session.tasks[1].depends_on["paths"]) == 1
+ result = runner.invoke(cli, [tmp_path.as_posix()])
+ assert result.exit_code == ExitCode.OK
+ assert "3 Collected tasks" in result.output
+ assert "3 Succeeded" in result.output
assert tmp_path.joinpath("a-copy.txt").exists()
diff --git a/tests/test_jupyter/test_task_generator.ipynb b/tests/test_jupyter/test_task_generator.ipynb
index 703352a5..1e61d8ea 100644
--- a/tests/test_jupyter/test_task_generator.ipynb
+++ b/tests/test_jupyter/test_task_generator.ipynb
@@ -24,7 +24,6 @@
"metadata": {},
"outputs": [],
"source": [
- "@task()\n",
"def task_create_files() -> Annotated[None, DirectoryNode(pattern=\"[ab].txt\")]:\n",
" path = Path()\n",
" path.joinpath(\"a.txt\").write_text(\"Hello, \")\n",
From d76c71663c6a6629a4f28f67c970fc90694506e8 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 17:36:36 +0100
Subject: [PATCH 72/81] no pragma. !
---
src/_pytask/nodes.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index a9485d5a..4babdf62 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -381,8 +381,8 @@ def signature(self) -> str:
def load(self, is_product: bool = False) -> Path:
if is_product:
return self.root_dir # type: ignore[return-value]
- msg = "'DirectoryNode' cannot be loaded as a dependency"
- raise NotImplementedError(msg) # pragma: no cover
+ msg = "'DirectoryNode' cannot be loaded as a dependency" # pragma: no cover
+ raise NotImplementedError(msg)
def collect(self) -> list[Path]:
"""Collect paths defined by the pattern."""
From c001a9d3d66ab8e75227d5c8bffc1444837a023c Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 17:47:40 +0100
Subject: [PATCH 73/81] Remove ways to generate tasks based on returns.
---
.../provisional_nodes_and_task_generators.md | 4 +
docs/source/reference_guides/api.md | 6 +-
.../repeating_tasks_with_different_inputs.md | 6 +-
docs/source/tutorials/selecting_tasks.md | 3 +-
docs/source/tutorials/skipping_tasks.md | 3 +-
docs/source/tutorials/write_a_task.md | 6 +-
src/_pytask/delayed.py | 31 +-----
tests/test_execute.py | 102 ------------------
8 files changed, 24 insertions(+), 137 deletions(-)
diff --git a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
index c63a8894..443f48a5 100644
--- a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
+++ b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -81,3 +81,7 @@ content to a `.txt` file.
```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_task_generator.py
```
+
+```{important}
+The generated tasks need to be decoratored with {func}`@task <pytask.task>` to be collected.
+```
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index 22872173..d25f7953 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,7 +136,8 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function(): ...
+def task_function():
+ ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -153,7 +154,8 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function(): ...
+def task_function():
+ ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index b369167b..8e5a10e5 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,7 +268,8 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces): ...
+ def task_create_random_data(seed, produces):
+ ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -288,7 +289,8 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces): ...
+ def task_create_random_data(i, produces):
+ ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 23b06903..266b47bf 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,7 +91,8 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i): ...
+ def task_parametrized(i=i):
+ ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index 13194d4c..6e511a5b 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -44,7 +44,8 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
def task_that_takes_really_long_to_run(
path: Path = Path("time_intensive_product.pkl"),
-): ...
+):
+ ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9ecc3bbc..9d949077 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,14 +117,16 @@ from pytask import task
@task
-def create_random_data(): ...
+def create_random_data():
+ ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data(): ...
+def create_random_data():
+ ...
```
## Customize task module names
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index 8eb6be63..daebaf51 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -23,7 +23,6 @@
from _pytask.task_utils import parse_collected_tasks_with_task_marker
from _pytask.tree_util import tree_map
from _pytask.tree_util import tree_map_with_path
-from _pytask.typing import is_task_function
from _pytask.typing import is_task_generator
from pytask import TaskOutcome
@@ -49,13 +48,8 @@ def _safe_load(node: PNode | PProvisionalNode, task: PTask, is_product: bool) ->
raise NodeLoadError(msg) from e
-_ERROR_TASK_GENERATOR_RETURN = """\
-Could not collect return of task generator. The return should be a task function or a \
-task class but received {} instead."""
-
-
@hookimpl
-def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, PLR0912
+def pytask_execute_task(session: Session, task: PTask) -> None:
"""Execute task generators and collect the tasks."""
if is_task_generator(task):
kwargs = {}
@@ -67,9 +61,7 @@ def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, P
if name in parameters:
kwargs[name] = tree_map(lambda x: _safe_load(x, task, True), value)
- out = task.execute(**kwargs)
- if inspect.isgenerator(out):
- out = list(out)
+ task.execute(**kwargs)
# Parse tasks created with @task.
name_to_function: Mapping[str, Callable[..., Any] | PTask]
@@ -80,23 +72,8 @@ def pytask_execute_task(session: Session, task: PTask) -> None: # noqa: C901, P
tasks = COLLECTED_TASKS.pop(None)
name_to_function = parse_collected_tasks_with_task_marker(tasks)
else:
- # Parse individual tasks.
- if is_task_function(out) or isinstance(out, PTask):
- out = [out]
- # Parse tasks from iterable.
- if hasattr(out, "__iter__"):
- name_to_function = {}
- for obj in out:
- if is_task_function(obj):
- name_to_function[obj.__name__] = obj
- elif isinstance(obj, PTask):
- name_to_function[obj.name] = obj
- else:
- msg = _ERROR_TASK_GENERATOR_RETURN.format(obj)
- raise ValueError(msg)
- else:
- msg = _ERROR_TASK_GENERATOR_RETURN.format(out)
- raise ValueError(msg)
+ msg = "The task generator {task.name!r} did not create any tasks."
+ raise RuntimeError(msg)
new_reports = []
for name, function in name_to_function.items():
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 80e85f96..23b9c50b 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -1093,108 +1093,6 @@ def task_copy(
assert tmp_path.joinpath("b-copy.txt").exists()
-@pytest.mark.end_to_end()
-def test_delayed_task_generation_with_generator(runner, tmp_path):
- source = """
- from typing_extensions import Annotated
- from pytask import DirectoryNode, task
- from pathlib import Path
-
- def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]:
- path = Path(__file__).parent
- path.joinpath("a.txt").write_text("Hello, ")
- path.joinpath("b.txt").write_text("World!")
-
- @task(after=task_produces, is_generator=True)
- def task_depends(
- paths = DirectoryNode(pattern="[ab].txt")
- ) -> ...:
- for path in paths:
-
- @task
- def task_copy(
- path: Path = path
- ) -> Annotated[str, path.with_name(path.stem + "-copy.txt")]:
- return path.read_text()
-
- yield task_copy
- """
- tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
-
- result = runner.invoke(cli, [tmp_path.as_posix()])
- assert result.exit_code == ExitCode.OK
- assert "4 Collected tasks" in result.output
- assert "4 Succeeded" in result.output
- assert tmp_path.joinpath("a-copy.txt").exists()
- assert tmp_path.joinpath("b-copy.txt").exists()
-
-
-@pytest.mark.end_to_end()
-def test_delayed_task_generation_with_single_function(runner, tmp_path):
- source = """
- from typing_extensions import Annotated
- from pytask import DirectoryNode, task
- from pathlib import Path
-
- def task_produces() -> Annotated[None, DirectoryNode(pattern="[a].txt")]:
- path = Path(__file__).parent
- path.joinpath("a.txt").write_text("Hello, ")
-
- @task(after=task_produces, is_generator=True)
- def task_depends(
- paths = DirectoryNode(pattern="[a].txt")
- ) -> ...:
- path = paths[0]
-
- def task_copy(
- path: Path = path
- ) -> Annotated[str, path.with_name(path.stem + "-copy.txt")]:
- return path.read_text()
- return task_copy
- """
- tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
-
- result = runner.invoke(cli, [tmp_path.as_posix()])
- assert result.exit_code == ExitCode.OK
- assert "3 Collected tasks" in result.output
- assert "3 Succeeded" in result.output
- assert tmp_path.joinpath("a-copy.txt").exists()
-
-
-@pytest.mark.end_to_end()
-def test_delayed_task_generation_with_task_node(runner, tmp_path):
- source = """
- from typing_extensions import Annotated
- from pytask import DirectoryNode, TaskWithoutPath, task, PathNode
- from pathlib import Path
-
- def task_produces() -> Annotated[None, DirectoryNode(pattern="[a].txt")]:
- path = Path(__file__).parent
- path.joinpath("a.txt").write_text("Hello, ")
-
- @task(after=task_produces, is_generator=True)
- def task_depends(
- paths = DirectoryNode(pattern="[a].txt")
- ) -> ...:
- path = paths[0]
-
- task_copy = TaskWithoutPath(
- name="task_copy",
- function=lambda path: path.read_text(),
- depends_on={"path": PathNode(path=path)},
- produces={"return": PathNode(path=path.with_name(path.stem + "-copy.txt"))},
- )
- return task_copy
- """
- tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
-
- result = runner.invoke(cli, [tmp_path.as_posix()])
- assert result.exit_code == ExitCode.OK
- assert "3 Collected tasks" in result.output
- assert "3 Succeeded" in result.output
- assert tmp_path.joinpath("a-copy.txt").exists()
-
-
@pytest.mark.end_to_end()
def test_gracefully_fail_when_task_generator_raises_error(runner, tmp_path):
source = """
From f80d3ef7addad32c80fc0d8dd97b1225c359ac26 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
<66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 4 Feb 2024 16:48:33 +0000
Subject: [PATCH 74/81] [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
---
docs/source/reference_guides/api.md | 6 ++----
.../tutorials/repeating_tasks_with_different_inputs.md | 6 ++----
docs/source/tutorials/selecting_tasks.md | 3 +--
docs/source/tutorials/skipping_tasks.md | 3 +--
docs/source/tutorials/write_a_task.md | 6 ++----
5 files changed, 8 insertions(+), 16 deletions(-)
diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md
index d25f7953..22872173 100644
--- a/docs/source/reference_guides/api.md
+++ b/docs/source/reference_guides/api.md
@@ -136,8 +136,7 @@ For example:
```python
@pytask.mark.timeout(10, "slow", method="thread")
-def task_function():
- ...
+def task_function(): ...
```
Will create and attach a {class}`Mark <pytask.Mark>` object to the collected
@@ -154,8 +153,7 @@ Example for using multiple custom markers:
```python
@pytask.mark.timeout(10, "slow", method="thread")
@pytask.mark.slow
-def task_function():
- ...
+def task_function(): ...
```
### Classes
diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
index 8e5a10e5..b369167b 100644
--- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md
+++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md
@@ -268,8 +268,7 @@ the {func}`@task <pytask.task>` decorator to pass keyword arguments to the task.
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(seed, produces):
- ...
+ def task_create_random_data(seed, produces): ...
```
Writing a function that creates `ID_TO_KWARGS` would be even more pythonic.
@@ -289,8 +288,7 @@ ID_TO_KWARGS = create_parametrization()
for id_, kwargs in ID_TO_KWARGS.items():
@task(id=id_, kwargs=kwargs)
- def task_create_random_data(i, produces):
- ...
+ def task_create_random_data(i, produces): ...
```
The {doc}`best-practices guide on parametrizations <../how_to_guides/bp_scaling_tasks>`
diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md
index 266b47bf..23b06903 100644
--- a/docs/source/tutorials/selecting_tasks.md
+++ b/docs/source/tutorials/selecting_tasks.md
@@ -91,8 +91,7 @@ from pytask import task
for i in range(2):
@task
- def task_parametrized(i=i):
- ...
+ def task_parametrized(i=i): ...
```
To run the task where `i = 1`, run this command.
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index 6e511a5b..13194d4c 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -44,8 +44,7 @@ from config import NO_LONG_RUNNING_TASKS
@pytask.mark.skipif(NO_LONG_RUNNING_TASKS, reason="Skip long-running tasks.")
def task_that_takes_really_long_to_run(
path: Path = Path("time_intensive_product.pkl"),
-):
- ...
+): ...
```
## Further reading
diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md
index 9d949077..9ecc3bbc 100644
--- a/docs/source/tutorials/write_a_task.md
+++ b/docs/source/tutorials/write_a_task.md
@@ -117,16 +117,14 @@ from pytask import task
@task
-def create_random_data():
- ...
+def create_random_data(): ...
# The id will be ".../task_data_preparation.py::create_data".
@task(name="create_data")
-def create_random_data():
- ...
+def create_random_data(): ...
```
## Customize task module names
From 315f354edc3dacdf124ec12c682c680ef56769a4 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 4 Feb 2024 17:54:36 +0100
Subject: [PATCH 75/81] Fix.
---
src/_pytask/collect.py | 2 +-
src/_pytask/nodes.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py
index 55a35bc1..7a96d484 100644
--- a/src/_pytask/collect.py
+++ b/src/_pytask/collect.py
@@ -437,7 +437,7 @@ def pytask_collect_node( # noqa: C901, PLR0912
node.name = create_name_of_python_node(node_info)
return node
- if isinstance(node, UPath):
+ if isinstance(node, UPath): # pragma: no cover
if not node.protocol:
node = Path(node)
else:
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index 4babdf62..3e01d833 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -382,7 +382,7 @@ def load(self, is_product: bool = False) -> Path:
if is_product:
return self.root_dir # type: ignore[return-value]
msg = "'DirectoryNode' cannot be loaded as a dependency" # pragma: no cover
- raise NotImplementedError(msg)
+ raise NotImplementedError(msg) # pragma: no cover
def collect(self) -> list[Path]:
"""Collect paths defined by the pattern."""
From ef42e6e98de5e1b1971ed7c6709dfb37c3000aa5 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 9 Mar 2024 13:32:08 +0100
Subject: [PATCH 76/81] Fix.
---
src/_pytask/delayed.py | 8 +++++---
src/_pytask/delayed_utils.py | 17 ++++++++++++-----
tests/conftest.py | 2 +-
tests/test_jupyter/test_task_generator.ipynb | 6 +++---
tests/test_task_utils.py | 5 ++---
5 files changed, 23 insertions(+), 15 deletions(-)
diff --git a/src/_pytask/delayed.py b/src/_pytask/delayed.py
index daebaf51..ffd3e496 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/delayed.py
@@ -1,17 +1,20 @@
"""Contains hook implementations for provisional nodes and task generators."""
+
from __future__ import annotations
import inspect
import sys
+from typing import TYPE_CHECKING
from typing import Any
from typing import Callable
from typing import Mapping
-from typing import TYPE_CHECKING
+
+from pytask import TaskOutcome
from _pytask.config import hookimpl
+from _pytask.delayed_utils import TASKS_WITH_PROVISIONAL_NODES
from _pytask.delayed_utils import collect_provisional_nodes
from _pytask.delayed_utils import recreate_dag
-from _pytask.delayed_utils import TASKS_WITH_PROVISIONAL_NODES
from _pytask.exceptions import NodeLoadError
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PProvisionalNode
@@ -24,7 +27,6 @@
from _pytask.tree_util import tree_map
from _pytask.tree_util import tree_map_with_path
from _pytask.typing import is_task_generator
-from pytask import TaskOutcome
if TYPE_CHECKING:
from _pytask.session import Session
diff --git a/src/_pytask/delayed_utils.py b/src/_pytask/delayed_utils.py
index e01f39b2..0844c2a3 100644
--- a/src/_pytask/delayed_utils.py
+++ b/src/_pytask/delayed_utils.py
@@ -2,11 +2,16 @@
import sys
from pathlib import Path
-from typing import Any
from typing import TYPE_CHECKING
+from typing import Any
from _pytask.collect_utils import collect_dependency
+from _pytask.dag import _check_if_dag_has_cycles
+from _pytask.dag import _check_if_tasks_have_the_same_products
+from _pytask.dag import _create_dag
+from _pytask.dag import _modify_dag
from _pytask.dag_utils import TopologicalSorter
+from _pytask.mark import select_tasks_by_marks_and_expressions
from _pytask.models import NodeInfo
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PProvisionalNode
@@ -69,10 +74,12 @@ def collect_provisional_nodes(
def recreate_dag(session: Session, task: PTask) -> None:
"""Recreate the DAG."""
try:
- session.dag = session.hook.pytask_dag_create_dag(
- session=session, tasks=session.tasks
- )
- session.hook.pytask_dag_modify_dag(session=session, dag=session.dag)
+ dag = _create_dag(tasks=session.tasks)
+ _check_if_dag_has_cycles(dag)
+ _check_if_tasks_have_the_same_products(dag, session.config["paths"])
+ _modify_dag(session=session, dag=dag)
+ select_tasks_by_marks_and_expressions(session=session, dag=dag)
+ session.dag = dag
session.scheduler = TopologicalSorter.from_dag_and_sorter(
session.dag, session.scheduler
)
diff --git a/tests/conftest.py b/tests/conftest.py
index a2380752..75a31412 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -29,7 +29,7 @@ def _remove_variable_info_from_output(data: str, path: Any) -> str: # noqa: ARG
# Remove dynamic versions.
index_root = next(i for i, line in enumerate(lines) if line.startswith("Root:"))
- new_info_line = "".join(lines[1:index_root])
+ new_info_line = " ".join(lines[1:index_root])
for platform in ("linux", "win32", "darwin"):
new_info_line = new_info_line.replace(platform, "<platform>")
pattern = re.compile(version.VERSION_PATTERN, flags=re.IGNORECASE | re.VERBOSE)
diff --git a/tests/test_jupyter/test_task_generator.ipynb b/tests/test_jupyter/test_task_generator.ipynb
index 1e61d8ea..2ef6aa61 100644
--- a/tests/test_jupyter/test_task_generator.ipynb
+++ b/tests/test_jupyter/test_task_generator.ipynb
@@ -3,7 +3,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "12bc75b1",
+ "id": "0",
"metadata": {},
"outputs": [],
"source": [
@@ -20,7 +20,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "29ac7311",
+ "id": "1",
"metadata": {},
"outputs": [],
"source": [
@@ -46,7 +46,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "738c9418",
+ "id": "2",
"metadata": {},
"outputs": [],
"source": [
diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py
index 47bb14a5..37e3cc44 100644
--- a/tests/test_task_utils.py
+++ b/tests/test_task_utils.py
@@ -6,10 +6,10 @@
from typing import NamedTuple
import pytest
+from _pytask.task_utils import COLLECTED_TASKS
from _pytask.task_utils import _arg_value_to_id_component
from _pytask.task_utils import _parse_name
from _pytask.task_utils import _parse_task_kwargs
-from _pytask.task_utils import COLLECTED_TASKS
from attrs import define
from pytask import Mark
from pytask import task
@@ -67,8 +67,7 @@ def test_parse_task_kwargs(kwargs, expectation, expected):
@pytest.mark.integration()
def test_default_values_of_pytask_meta():
@task()
- def task_example():
- ...
+ def task_example(): ...
assert task_example.pytask_meta.after == []
assert not task_example.pytask_meta.is_generator
From eacecf443cf44a0119cdfc516d5f4e8630263bd6 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sat, 9 Mar 2024 14:36:27 +0100
Subject: [PATCH 77/81] Change.
---
docs/source/changes.md | 2 +-
.../provisional_nodes_and_task_generators.md | 19 ++++++++++---------
...ed_products.py => provisional_products.py} | 0
...ks_delayed_task.py => provisional_task.py} | 0
...rator.py => provisional_task_generator.py} | 0
src/_pytask/execute.py | 2 +-
src/_pytask/node_protocols.py | 2 +-
src/_pytask/persist.py | 2 +-
src/_pytask/pluginmanager.py | 2 +-
src/_pytask/{delayed.py => provisional.py} | 6 +++---
...{delayed_utils.py => provisional_utils.py} | 0
src/_pytask/skipping.py | 2 +-
tests/test_collect_command.py | 4 ++--
tests/test_execute.py | 12 ++++++------
14 files changed, 27 insertions(+), 26 deletions(-)
rename docs_src/how_to_guides/{delayed_tasks_delayed_products.py => provisional_products.py} (100%)
rename docs_src/how_to_guides/{delayed_tasks_delayed_task.py => provisional_task.py} (100%)
rename docs_src/how_to_guides/{delayed_tasks_task_generator.py => provisional_task_generator.py} (100%)
rename src/_pytask/{delayed.py => provisional.py} (96%)
rename src/_pytask/{delayed_utils.py => provisional_utils.py} (100%)
diff --git a/docs/source/changes.md b/docs/source/changes.md
index 3838a8e1..4c81a0eb 100644
--- a/docs/source/changes.md
+++ b/docs/source/changes.md
@@ -62,7 +62,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
- {pull}`485` adds missing steps to unconfigure pytask after the job is done, which
caused flaky tests.
- {pull}`486` adds default names to {class}`~pytask.PPathNode`.
-- {pull}`487` implements delayed tasks and nodes.
+- {pull}`487` implements task generators and provisional nodes.
- {pull}`488` raises an error when an invalid value is used in a return annotation.
- {pull}`489` and {pull}`491` simplifies parsing products and does not raise an error
when a product annotation is used with the argument name `produces`. And allow
diff --git a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
index 443f48a5..844187b8 100644
--- a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
+++ b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -21,7 +21,7 @@ The following sections will explain how you use pytask in these situations.
Let us start with a task that downloads all files without an extension from the root
folder of the pytask repository and stores them on disk in a folder called `downloads`.
-```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_delayed_products.py
+```{literalinclude} ../../../docs_src/how_to_guides/provisional_products.py
---
emphasize-lines: 4, 11
---
@@ -49,21 +49,21 @@ actual nodes. A {class}`~pytask.DirectoryNode`, for example, returns
In the next step, we want to define a task that consumes and merges all previously
downloaded files into one file.
-```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_delayed_task.py
+```{literalinclude} ../../../docs_src/how_to_guides/provisional_task.py
---
emphasize-lines: 9
---
```
-Here, we use the {class}`~pytask.DirectoryNode` as a dependency since we do not know the
+Here, the {class}`~pytask.DirectoryNode` is a dependency because we do not know the
names of the downloaded files. Before the task is executed, the list of files in the
folder defined by the root path and the pattern are automatically collected and passed
to the task.
-If we use a {class}`DirectoryNode` with the same `root_dir` and `pattern` in both tasks,
-pytask will automatically recognize that the second task depends on the first. If that
-is not true, you might need to make this dependency more explicit by using
-{func}`@task(after=...) <pytask.task>`, which is explained {ref}`here <after>`.
+If we use a {class}`~pytask.DirectoryNode` with the same `root_dir` and `pattern` in
+both tasks, pytask will automatically recognize that the second task depends on the
+first. If that is not true, you might need to make this dependency more explicit by
+using {func}`@task(after=...) <pytask.task>`, which is explained {ref}`here <after>`.
## Task generators
@@ -79,9 +79,10 @@ writing functions in a task module.
The code snippet shows each task takes one of the downloaded files and copies its
content to a `.txt` file.
-```{literalinclude} ../../../docs_src/how_to_guides/delayed_tasks_task_generator.py
+```{literalinclude} ../../../docs_src/how_to_guides/provisional_task_generator.py
```
```{important}
-The generated tasks need to be decoratored with {func}`@task <pytask.task>` to be collected.
+The generated tasks need to be decoratored with {func}`@task <pytask.task>` to be
+collected.
```
diff --git a/docs_src/how_to_guides/delayed_tasks_delayed_products.py b/docs_src/how_to_guides/provisional_products.py
similarity index 100%
rename from docs_src/how_to_guides/delayed_tasks_delayed_products.py
rename to docs_src/how_to_guides/provisional_products.py
diff --git a/docs_src/how_to_guides/delayed_tasks_delayed_task.py b/docs_src/how_to_guides/provisional_task.py
similarity index 100%
rename from docs_src/how_to_guides/delayed_tasks_delayed_task.py
rename to docs_src/how_to_guides/provisional_task.py
diff --git a/docs_src/how_to_guides/delayed_tasks_task_generator.py b/docs_src/how_to_guides/provisional_task_generator.py
similarity index 100%
rename from docs_src/how_to_guides/delayed_tasks_task_generator.py
rename to docs_src/how_to_guides/provisional_task_generator.py
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index cda1a4db..c3a4e110 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -22,7 +22,6 @@
from _pytask.dag_utils import node_and_neighbors
from _pytask.database_utils import has_node_changed
from _pytask.database_utils import update_states_in_database
-from _pytask.delayed_utils import collect_provisional_products
from _pytask.exceptions import ExecutionError
from _pytask.exceptions import NodeLoadError
from _pytask.exceptions import NodeNotFoundError
@@ -38,6 +37,7 @@
from _pytask.outcomes import WouldBeExecuted
from _pytask.outcomes import count_outcomes
from _pytask.pluginmanager import hookimpl
+from _pytask.provisional_utils import collect_provisional_products
from _pytask.reports import ExecutionReport
from _pytask.traceback import remove_traceback_from_exc_info
from _pytask.tree_util import tree_leaves
diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py
index ad64c76a..6a1d8fc0 100644
--- a/src/_pytask/node_protocols.py
+++ b/src/_pytask/node_protocols.py
@@ -129,7 +129,7 @@ def load(self, is_product: bool = False) -> Any: # pragma: no cover
It is possible to load a provisional node as a dependency so that it can inject
basic information about it in the task. For example,
- :meth:`pytask.DirectoryNode` injects the root directory.
+ :meth:`pytask.DirectoryNode.load` injects the root directory.
"""
if is_product:
diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py
index bb2e653a..8709dc0d 100644
--- a/src/_pytask/persist.py
+++ b/src/_pytask/persist.py
@@ -8,11 +8,11 @@
from _pytask.dag_utils import node_and_neighbors
from _pytask.database_utils import has_node_changed
from _pytask.database_utils import update_states_in_database
-from _pytask.delayed_utils import collect_provisional_products
from _pytask.mark_utils import has_mark
from _pytask.outcomes import Persisted
from _pytask.outcomes import TaskOutcome
from _pytask.pluginmanager import hookimpl
+from _pytask.provisional_utils import collect_provisional_products
if TYPE_CHECKING:
from _pytask.node_protocols import PTask
diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py
index 6093278a..4674c28b 100644
--- a/src/_pytask/pluginmanager.py
+++ b/src/_pytask/pluginmanager.py
@@ -46,7 +46,7 @@ def pytask_add_hooks(pm: PluginManager) -> None:
"_pytask.dag_command",
"_pytask.database",
"_pytask.debugging",
- "_pytask.delayed",
+ "_pytask.provisional",
"_pytask.execute",
"_pytask.live",
"_pytask.logging",
diff --git a/src/_pytask/delayed.py b/src/_pytask/provisional.py
similarity index 96%
rename from src/_pytask/delayed.py
rename to src/_pytask/provisional.py
index ffd3e496..084aabd2 100644
--- a/src/_pytask/delayed.py
+++ b/src/_pytask/provisional.py
@@ -12,15 +12,15 @@
from pytask import TaskOutcome
from _pytask.config import hookimpl
-from _pytask.delayed_utils import TASKS_WITH_PROVISIONAL_NODES
-from _pytask.delayed_utils import collect_provisional_nodes
-from _pytask.delayed_utils import recreate_dag
from _pytask.exceptions import NodeLoadError
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PProvisionalNode
from _pytask.node_protocols import PTask
from _pytask.node_protocols import PTaskWithPath
from _pytask.outcomes import CollectionOutcome
+from _pytask.provisional_utils import TASKS_WITH_PROVISIONAL_NODES
+from _pytask.provisional_utils import collect_provisional_nodes
+from _pytask.provisional_utils import recreate_dag
from _pytask.reports import ExecutionReport
from _pytask.task_utils import COLLECTED_TASKS
from _pytask.task_utils import parse_collected_tasks_with_task_marker
diff --git a/src/_pytask/delayed_utils.py b/src/_pytask/provisional_utils.py
similarity index 100%
rename from src/_pytask/delayed_utils.py
rename to src/_pytask/provisional_utils.py
diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py
index 6f8e5861..a7678154 100644
--- a/src/_pytask/skipping.py
+++ b/src/_pytask/skipping.py
@@ -6,7 +6,6 @@
from typing import Any
from _pytask.dag_utils import descending_tasks
-from _pytask.delayed_utils import collect_provisional_products
from _pytask.mark import Mark
from _pytask.mark_utils import get_marks
from _pytask.mark_utils import has_mark
@@ -15,6 +14,7 @@
from _pytask.outcomes import SkippedUnchanged
from _pytask.outcomes import TaskOutcome
from _pytask.pluginmanager import hookimpl
+from _pytask.provisional_utils import collect_provisional_products
if TYPE_CHECKING:
from _pytask.node_protocols import PTask
diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py
index 8fabcb29..1112c3da 100644
--- a/tests/test_collect_command.py
+++ b/tests/test_collect_command.py
@@ -640,7 +640,7 @@ def task_example() -> Annotated[Dict[str, str], nodes]:
") -> Annotated[None, DirectoryNode(pattern='*.txt')]",
],
)
-def test_collect_task_with_delayed_path_node_as_product(runner, tmp_path, node_def):
+def test_collect_task_with_provisional_path_node_as_product(runner, tmp_path, node_def):
source = f"""
from pytask import DirectoryNode, Product
from typing_extensions import Annotated, List
@@ -672,7 +672,7 @@ def task_example({node_def}: ...
@pytest.mark.end_to_end()
-def test_collect_task_with_delayed_dependencies(runner, tmp_path):
+def test_collect_task_with_provisional_dependencies(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode
diff --git a/tests/test_execute.py b/tests/test_execute.py
index 51ef71d7..8459914a 100644
--- a/tests/test_execute.py
+++ b/tests/test_execute.py
@@ -927,7 +927,7 @@ def func(path): path.touch()
@pytest.mark.end_to_end()
-def test_task_that_produces_delayed_path_node(tmp_path):
+def test_task_that_produces_provisional_path_node(tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, Product
@@ -953,7 +953,7 @@ def task_example(
@pytest.mark.end_to_end()
-def test_task_that_depends_on_relative_delayed_path_node(tmp_path):
+def test_task_that_depends_on_relative_provisional_path_node(tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode
@@ -977,7 +977,7 @@ def task_example(
@pytest.mark.end_to_end()
-def test_task_that_depends_on_delayed_path_node_with_root_dir(tmp_path):
+def test_task_that_depends_on_provisional_path_node_with_root_dir(tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode
@@ -1004,7 +1004,7 @@ def task_example(
@pytest.mark.end_to_end()
-def test_task_that_depends_on_delayed_task(runner, tmp_path):
+def test_task_that_depends_on_provisional_task(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, task
@@ -1060,7 +1060,7 @@ def task_depends(
@pytest.mark.end_to_end()
-def test_delayed_task_generation(runner, tmp_path):
+def test_provisional_task_generation(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, task
@@ -1115,7 +1115,7 @@ def task_example(
@pytest.mark.end_to_end()
-def test_use_delayed_node_as_product_in_generator_without_rerun(runner, tmp_path):
+def test_use_provisional_node_as_product_in_generator_without_rerun(runner, tmp_path):
source = """
from typing_extensions import Annotated
from pytask import DirectoryNode, task, Product
From 43486d7d6ad8bd00d7b84b888ce90cdf655b7b4d Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Sun, 10 Mar 2024 10:23:36 +0100
Subject: [PATCH 78/81] Fix.
---
src/_pytask/dag.py | 26 ++++++++++++++++----------
src/_pytask/provisional_utils.py | 22 ++++++++++------------
tests/test_dag.py | 4 ++--
3 files changed, 28 insertions(+), 24 deletions(-)
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
index 8377564f..d8c8b325 100644
--- a/src/_pytask/dag.py
+++ b/src/_pytask/dag.py
@@ -1,4 +1,4 @@
-"""Contains code related to resolving dependencies."""
+"""Contains code related to the DAG."""
from __future__ import annotations
@@ -34,18 +34,13 @@
from _pytask.session import Session
-__all__ = ["create_dag"]
+__all__ = ["create_dag", "create_dag_from_session"]
def create_dag(session: Session) -> nx.DiGraph:
"""Create a directed acyclic graph (DAG) for the workflow."""
try:
- dag = _create_dag(tasks=session.tasks)
- _check_if_dag_has_cycles(dag)
- _check_if_tasks_have_the_same_products(dag, session.config["paths"])
- _modify_dag(session=session, dag=dag)
- select_tasks_by_marks_and_expressions(session=session, dag=dag)
-
+ dag = create_dag_from_session(session)
except Exception: # noqa: BLE001
report = DagReport.from_exception(sys.exc_info())
_log_dag(report=report)
@@ -55,7 +50,17 @@ def create_dag(session: Session) -> nx.DiGraph:
return dag
-def _create_dag(tasks: list[PTask]) -> nx.DiGraph:
+def create_dag_from_session(session: Session) -> nx.DiGraph:
+ """Create a DAG from a session."""
+ dag = _create_dag_from_tasks(tasks=session.tasks)
+ _check_if_dag_has_cycles(dag)
+ _check_if_tasks_have_the_same_products(dag, session.config["paths"])
+ dag = _modify_dag(session=session, dag=dag)
+ select_tasks_by_marks_and_expressions(session=session, dag=dag)
+ return dag
+
+
+def _create_dag_from_tasks(tasks: list[PTask]) -> nx.DiGraph:
"""Create the DAG from tasks, dependencies and products."""
def _add_dependency(
@@ -98,7 +103,7 @@ def _add_product(
return dag
-def _modify_dag(session: Session, dag: nx.DiGraph) -> None:
+def _modify_dag(session: Session, dag: nx.DiGraph) -> nx.DiGraph:
"""Create dependencies between tasks when using ``@task(after=...)``."""
temporary_id_to_task = {
task.attributes["collection_id"]: task
@@ -119,6 +124,7 @@ def _modify_dag(session: Session, dag: nx.DiGraph) -> None:
for signature in signatures:
for successor in dag.successors(signature):
dag.add_edge(successor, task.signature)
+ return dag
def _check_if_dag_has_cycles(dag: nx.DiGraph) -> None:
diff --git a/src/_pytask/provisional_utils.py b/src/_pytask/provisional_utils.py
index 0844c2a3..9a09b096 100644
--- a/src/_pytask/provisional_utils.py
+++ b/src/_pytask/provisional_utils.py
@@ -1,3 +1,5 @@
+"""Contains utilities related to provisional nodes and task generators."""
+
from __future__ import annotations
import sys
@@ -6,12 +8,8 @@
from typing import Any
from _pytask.collect_utils import collect_dependency
-from _pytask.dag import _check_if_dag_has_cycles
-from _pytask.dag import _check_if_tasks_have_the_same_products
-from _pytask.dag import _create_dag
-from _pytask.dag import _modify_dag
+from _pytask.dag import create_dag_from_session
from _pytask.dag_utils import TopologicalSorter
-from _pytask.mark import select_tasks_by_marks_and_expressions
from _pytask.models import NodeInfo
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PProvisionalNode
@@ -72,14 +70,14 @@ def collect_provisional_nodes(
def recreate_dag(session: Session, task: PTask) -> None:
- """Recreate the DAG."""
+ """Recreate the DAG when provisional nodes are resolved.
+
+ If the DAG resolution fails, the error is attached as an execution report since
+ there is not better mechanic yet to display the error.
+
+ """
try:
- dag = _create_dag(tasks=session.tasks)
- _check_if_dag_has_cycles(dag)
- _check_if_tasks_have_the_same_products(dag, session.config["paths"])
- _modify_dag(session=session, dag=dag)
- select_tasks_by_marks_and_expressions(session=session, dag=dag)
- session.dag = dag
+ session.dag = create_dag_from_session(session)
session.scheduler = TopologicalSorter.from_dag_and_sorter(
session.dag, session.scheduler
)
diff --git a/tests/test_dag.py b/tests/test_dag.py
index 5272e8af..80c2bab5 100644
--- a/tests/test_dag.py
+++ b/tests/test_dag.py
@@ -5,7 +5,7 @@
from pathlib import Path
import pytest
-from _pytask.dag import _create_dag
+from _pytask.dag import _create_dag_from_tasks
from pytask import ExitCode
from pytask import PathNode
from pytask import Task
@@ -26,7 +26,7 @@ def test_create_dag():
1: PathNode.from_path(root / "node_2"),
},
)
- dag = _create_dag(tasks=[task])
+ dag = _create_dag_from_tasks(tasks=[task])
for signature in (
"90bb899a1b60da28ff70352cfb9f34a8bed485597c7f40eed9bd4c6449147525",
From c3b559644db457bbcff73baf67f7cdb59660d808 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Wed, 13 Mar 2024 10:42:38 +0100
Subject: [PATCH 79/81] Improve docs.
---
.../provisional_nodes_and_task_generators.md | 32 +++++++++++--------
.../how_to_guides/provisional_products.py | 24 +++++++++-----
2 files changed, 35 insertions(+), 21 deletions(-)
diff --git a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
index 844187b8..d5530eee 100644
--- a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
+++ b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -8,9 +8,9 @@ pytask's execution model can usually be separated into three phases.
But, in some situations, pytask needs to be more flexible.
-Imagine you want to download files from an online storage, but the total number of files
-and their filenames is unknown before the task has started. How can you still describe
-the files as products of the task?
+Imagine you want to download a folder with files from an online storage. Before the task
+is completed you do not know the total number of files or their filenames. How can you
+still describe the files as products of the task?
And how would you define another task that depends on these files?
@@ -18,19 +18,22 @@ The following sections will explain how you use pytask in these situations.
## Producing provisional nodes
-Let us start with a task that downloads all files without an extension from the root
-folder of the pytask repository and stores them on disk in a folder called `downloads`.
+As an example for the aforementioned scenario, let us write a task that downloads all
+files without a file extension from the root folder of the pytask GitHub repository. The
+files are downloaded to a folder called `downloads`. `downloads` is in the same folder
+as the task module because it is a relative path.
```{literalinclude} ../../../docs_src/how_to_guides/provisional_products.py
---
-emphasize-lines: 4, 11
+emphasize-lines: 4, 22
---
```
Since the names of the files are not known when pytask is started, we need to use a
-{class}`~pytask.DirectoryNode`. With a {class}`~pytask.DirectoryNode` we can specify
-where pytask can find the files. The files are described with a path (default is the
-directory of the task module) and a glob pattern (default is `*`).
+{class}`~pytask.DirectoryNode` to define the task's product. With a
+{class}`~pytask.DirectoryNode` we can specify where pytask can find the files. The files
+are described with a root path (default is the directory of the task module) and a glob
+pattern (default is `*`).
When we use the {class}`~pytask.DirectoryNode` as a product annotation, we get access to
the `root_dir` as a {class}`~pathlib.Path` object inside the function, which allows us
@@ -49,16 +52,19 @@ actual nodes. A {class}`~pytask.DirectoryNode`, for example, returns
In the next step, we want to define a task that consumes and merges all previously
downloaded files into one file.
+The difficulty here is how can we reference the downloaded files before they have been
+downloaded.
+
```{literalinclude} ../../../docs_src/how_to_guides/provisional_task.py
---
emphasize-lines: 9
---
```
-Here, the {class}`~pytask.DirectoryNode` is a dependency because we do not know the
-names of the downloaded files. Before the task is executed, the list of files in the
-folder defined by the root path and the pattern are automatically collected and passed
-to the task.
+To reference the files that will be downloaded, we use the
+{class}`~pytask.DirectoryNode` is a dependency. Before the task is executed, the list of
+files in the folder defined by the root path and the pattern are automatically collected
+and passed to the task.
If we use a {class}`~pytask.DirectoryNode` with the same `root_dir` and `pattern` in
both tasks, pytask will automatically recognize that the second task depends on the
diff --git a/docs_src/how_to_guides/provisional_products.py b/docs_src/how_to_guides/provisional_products.py
index 5b6454f5..269c327a 100644
--- a/docs_src/how_to_guides/provisional_products.py
+++ b/docs_src/how_to_guides/provisional_products.py
@@ -1,25 +1,33 @@
from pathlib import Path
-import requests
+import httpx
from pytask import DirectoryNode
from pytask import Product
from typing_extensions import Annotated
+def get_files_without_file_extensions_from_repo() -> list[str]:
+ url = "https://api.github.com/repos/pytask-dev/pytask/git/trees/main"
+ response = httpx.get(url)
+ elements = response.json()["tree"]
+ return [
+ e["path"]
+ for e in elements
+ if e["type"] == "blob" and Path(e["path"]).suffix == ""
+ ]
+
+
def task_download_files(
download_folder: Annotated[
Path, DirectoryNode(root_dir=Path("downloads"), pattern="*"), Product
],
) -> None:
"""Download files."""
- # Scrape list of files without file extension from
- # https://github.com/pytask-dev/pytask. (We skip this part for simplicity.)
- files_to_download = ("CITATION", "LICENSE")
+ # Contains names like CITATION or LICENSE.
+ files_to_download = get_files_without_file_extensions_from_repo()
- # Download them.
for file_ in files_to_download:
- response = requests.get(
- url=f"raw.githubusercontent.com/pytask-dev/pytask/main/{file_}", timeout=5
- )
+ url = "raw.githubusercontent.com/pytask-dev/pytask/main"
+ response = httpx.get(url=f"{url}/{file_}", timeout=5)
content = response.text
download_folder.joinpath(file_).write_text(content)
From bf95f75563e3fe79b7e5f3ac99e94e5d286c49b1 Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 15 Mar 2024 17:38:35 +0100
Subject: [PATCH 80/81] Fix.
---
src/_pytask/execute.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
index 523c950f..3a56a82e 100644
--- a/src/_pytask/execute.py
+++ b/src/_pytask/execute.py
@@ -140,6 +140,13 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C
node = dag.nodes[node_signature].get("task") or dag.nodes[
node_signature
].get("node")
+
+ # Skip provisional nodes that are products since they do not have a state.
+ if node_signature not in predecessors and isinstance(
+ node, PProvisionalNode
+ ):
+ continue
+
node_state = node.state()
if node_signature in predecessors and not node_state:
msg = f"{task.name!r} requires missing node {node.name!r}."
@@ -150,12 +157,6 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C
)
raise NodeNotFoundError(msg)
- # Skip provisional nodes that are products since they do not have a state.
- if node_signature not in predecessors and isinstance(
- node, PProvisionalNode
- ):
- continue
-
has_changed = has_node_changed(task=task, node=node, state=node_state)
if has_changed:
needs_to_be_executed = True
From 6e5679c8a9e8e8be6eac326ea50e646d19f94e5c Mon Sep 17 00:00:00 2001
From: Tobias Raabe <raabe@posteo.de>
Date: Fri, 15 Mar 2024 17:54:28 +0100
Subject: [PATCH 81/81] Last fixes.
---
docs/source/tutorials/skipping_tasks.md | 2 +-
src/_pytask/nodes.py | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md
index 13194d4c..1ad3b38c 100644
--- a/docs/source/tutorials/skipping_tasks.md
+++ b/docs/source/tutorials/skipping_tasks.md
@@ -14,7 +14,7 @@ skip tasks during development that take too much time to compute right now.
```
Not only will this task be skipped, but all tasks depending on
-`time_intensive_product`.pkl\`.
+`time_intensive_product.pkl`.
## Conditional skipping
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
index c65c80c3..e7674e0f 100644
--- a/src/_pytask/nodes.py
+++ b/src/_pytask/nodes.py
@@ -359,6 +359,7 @@ def signature(self) -> str:
return hashlib.sha256(raw_key.encode()).hexdigest()
def load(self, is_product: bool = False) -> Path:
+ """Inject a path into the task when loaded as a product."""
if is_product:
return self.root_dir # type: ignore[return-value]
msg = "'DirectoryNode' cannot be loaded as a dependency" # pragma: no cover