Merge branch 'main' into uv

tobiasraabe · web-flow · commit 9479410cfc54 · 2024-03-15T17:17:07.000+01:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -5,9 +5,6 @@ concurrency:
   group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
-env:
-  CONDA_EXE: mamba
-
 on:
   push:
     branches:
diff --git a/.github/workflows/update-plugin-list.yml b/.github/workflows/update-plugin-list.yml
@@ -37,7 +37,7 @@ jobs:
         run: python scripts/update_plugin_list.py
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@b1ddad2c994a25fbc81a28b3ec0e368bb2021c50
+        uses: peter-evans/create-pull-request@a4f52f8033a6168103c2538976c07b467e8163bc
         with:
           commit-message: '[automated] Update plugin list'
           author: 'Tobias Raabe <tobiasraabe@users.noreply.github.com>'
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -33,18 +33,18 @@ repos:
     hooks:
     -   id: sort-all
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.1
+    rev: v0.3.2
     hooks:
       - id: ruff-format
       - id: ruff
         args: [--unsafe-fixes]
 -   repo: https://github.com/dosisod/refurb
-    rev: v1.28.0
+    rev: v2.0.0
     hooks:
     -   id: refurb
         args: [--ignore, FURB126]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.9.0
     hooks:
     -   id: mypy
         additional_dependencies: [
@@ -80,8 +80,6 @@ repos:
 -   repo: https://github.com/nbQA-dev/nbQA
     rev: 1.8.4
     hooks:
-    -   id: nbqa-black
-    -   id: nbqa-isort
     -   id: nbqa-mypy
         args: [--ignore-missing-imports]
 -   repo: https://github.com/kynan/nbstripout
diff --git a/docs/source/changes.md b/docs/source/changes.md
@@ -19,6 +19,11 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
 - {pull}`557` fixes an issue with `@task(after=...)` in notebooks and terminals.
 - {pull}`566` makes universal-pathlib an official dependency.
 - {pull}`567` adds uv to the CI workflow for faster installation.
+- {pull}`568` restricts `task_files` to a list of patterns and raises a better error.
+- {pull}`569` removes the hooks related to the creation of the DAG.
+- {pull}`571` removes redundant calls to `PNode.state()` which causes a high penalty for
+  remote files.
+- {pull}`573` removes the `pytask_execute_create_scheduler` hook.
 
 ## 0.4.5 - 2024-01-09
 
diff --git a/docs/source/reference_guides/configuration.md b/docs/source/reference_guides/configuration.md
@@ -242,7 +242,7 @@ strict_markers = true
 Change the pattern which identify task files.
 
 ```toml
-task_files = "task_*.py"  # default
+task_files = ["task_*.py"]  # default
 
 task_files = ["task_*.py", "tasks_*.py"]
 ```
diff --git a/docs/source/reference_guides/hookspecs.md b/docs/source/reference_guides/hookspecs.md
@@ -62,33 +62,13 @@ The following hooks traverse directories and collect tasks from files.
 .. autofunction:: pytask_collect_log
 ```
 
-## Resolving Dependencies
-
-The following hooks are designed to build a DAG from tasks and dependencies and check
-which files have changed and need to be re-run.
-
-```{warning}
-This step is still experimental and likely to change in the future. If you are planning
-to write a plugin which extends pytask in this dimension, please, start a discussion
-before writing a plugin. It may make your life easier if changes in pytask anticipate
-your plugin.
-```
-
-```{eval-rst}
-.. autofunction:: pytask_dag
-.. autofunction:: pytask_dag_create_dag
-.. autofunction:: pytask_dag_log
-
-```
-
 ## Execution
 
 The following hooks execute the tasks and log information on the result in the terminal.
 
 ```{eval-rst}
 .. autofunction:: pytask_execute
 .. autofunction:: pytask_execute_log_start
-.. autofunction:: pytask_execute_create_scheduler
 .. autofunction:: pytask_execute_build
 .. autofunction:: pytask_execute_task_protocol
 .. autofunction:: pytask_execute_task_log_start
diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md
@@ -72,7 +72,7 @@ task has finished, pytask will check whether the file exists.
 
 ````
 
-````{tab-item} prodouces
+````{tab-item} produces
 :sync: produces
 
 ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_products_produces.py
@@ -132,7 +132,7 @@ annotation are dependencies of the task.
 
 ````
 
-````{tab-item} prodouces
+````{tab-item} produces
 :sync: produces
 
 To specify that the task relies on the data set `data.pkl`, you can add the path to the
@@ -178,7 +178,7 @@ are assumed to point to a location relative to the task module.
 
 ````
 
-````{tab-item} prodouces
+````{tab-item} produces
 :sync: produces
 
 ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_relative_produces.py
diff --git a/src/_pytask/_hashlib.py b/src/_pytask/_hashlib.py
@@ -232,5 +232,5 @@ def hash_value(value: Any) -> int | str:
     if isinstance(value, str):
         value = value.encode()
     if isinstance(value, bytes):
-        return str(hashlib.sha256(value).hexdigest())
+        return hashlib.sha256(value).hexdigest()
     return hash(value)
diff --git a/src/_pytask/build.py b/src/_pytask/build.py
@@ -21,6 +21,7 @@
 from _pytask.config_utils import find_project_root_and_config
 from _pytask.config_utils import read_config
 from _pytask.console import console
+from _pytask.dag import create_dag
 from _pytask.exceptions import CollectionError
 from _pytask.exceptions import ConfigurationError
 from _pytask.exceptions import ExecutionError
@@ -95,7 +96,7 @@ def build(  # noqa: C901, PLR0912, PLR0913, PLR0915
     stop_after_first_failure: bool = False,
     strict_markers: bool = False,
     tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (),
-    task_files: str | Iterable[str] = "task_*.py",
+    task_files: Iterable[str] = ("task_*.py",),
     trace: bool = False,
     verbose: int = 1,
     **kwargs: Any,
@@ -265,7 +266,7 @@ def build(  # noqa: C901, PLR0912, PLR0913, PLR0915
         try:
             session.hook.pytask_log_session_header(session=session)
             session.hook.pytask_collect(session=session)
-            session.hook.pytask_dag(session=session)
+            session.dag = create_dag(session=session)
             session.hook.pytask_execute(session=session)
 
         except CollectionError:
diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py
@@ -19,6 +19,7 @@
 from _pytask.console import create_url_style_for_path
 from _pytask.console import format_node_name
 from _pytask.console import format_task_name
+from _pytask.dag import create_dag
 from _pytask.exceptions import CollectionError
 from _pytask.exceptions import ConfigurationError
 from _pytask.exceptions import ResolvingDependenciesError
@@ -70,7 +71,7 @@ def collect(**raw_config: Any | None) -> NoReturn:
         try:
             session.hook.pytask_log_session_header(session=session)
             session.hook.pytask_collect(session=session)
-            session.hook.pytask_dag(session=session)
+            session.dag = create_dag(session=session)
 
             tasks = _select_tasks_by_expressions_and_marker(session)
             task_with_path = [t for t in tasks if isinstance(t, PTaskWithPath)]
diff --git a/src/_pytask/config.py b/src/_pytask/config.py
@@ -92,7 +92,13 @@ def pytask_parse_config(config: dict[str, Any]) -> None:
         + IGNORED_TEMPORARY_FILES_AND_FOLDERS
     )
 
-    config["task_files"] = to_list(config.get("task_files", "task_*.py"))
+    value = config.get("task_files", ["task_*.py"])
+    if not isinstance(value, (list, tuple)) or not all(
+        isinstance(p, str) for p in value
+    ):
+        msg = "'task_files' must be a list of patterns."
+        raise ValueError(msg)
+    config["task_files"] = value
 
     if config["stop_after_first_failure"]:
         config["max_failures"] = 1
diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py
@@ -19,10 +19,10 @@
 from _pytask.console import render_to_string
 from _pytask.exceptions import ResolvingDependenciesError
 from _pytask.mark import select_by_after_keyword
+from _pytask.mark import select_tasks_by_marks_and_expressions
 from _pytask.node_protocols import PNode
 from _pytask.node_protocols import PTask
 from _pytask.nodes import PythonNode
-from _pytask.pluginmanager import hookimpl
 from _pytask.reports import DagReport
 from _pytask.shared import reduce_names_of_multiple_nodes
 from _pytask.tree_util import tree_map
@@ -33,28 +33,28 @@
     from _pytask.session import Session
 
 
-@hookimpl
-def pytask_dag(session: Session) -> bool | None:
+__all__ = ["create_dag"]
+
+
+def create_dag(session: Session) -> nx.DiGraph:
     """Create a directed acyclic graph (DAG) for the workflow."""
     try:
-        session.dag = session.hook.pytask_dag_create_dag(
-            session=session, tasks=session.tasks
-        )
-        session.hook.pytask_dag_modify_dag(session=session, dag=session.dag)
+        dag = _create_dag(tasks=session.tasks)
+        _check_if_dag_has_cycles(dag)
+        _check_if_tasks_have_the_same_products(dag, session.config["paths"])
+        _modify_dag(session=session, dag=dag)
+        select_tasks_by_marks_and_expressions(session=session, dag=dag)
 
     except Exception:  # noqa: BLE001
         report = DagReport.from_exception(sys.exc_info())
-        session.hook.pytask_dag_log(session=session, report=report)
+        _log_dag(report=report)
         session.dag_report = report
 
         raise ResolvingDependenciesError from None
-
-    else:
-        return True
+    return dag
 
 
-@hookimpl
-def pytask_dag_create_dag(session: Session, tasks: list[PTask]) -> nx.DiGraph:
+def _create_dag(tasks: list[PTask]) -> nx.DiGraph:
     """Create the DAG from tasks, dependencies and products."""
 
     def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
@@ -90,15 +90,10 @@ def _add_product(dag: nx.DiGraph, task: PTask, node: PNode) -> None:
             else None,
             task.depends_on,
         )
-
-    _check_if_dag_has_cycles(dag)
-    _check_if_tasks_have_the_same_products(dag, session.config["paths"])
-
     return dag
 
 
-@hookimpl
-def pytask_dag_modify_dag(session: Session, dag: nx.DiGraph) -> None:
+def _modify_dag(session: Session, dag: nx.DiGraph) -> None:
     """Create dependencies between tasks when using ``@task(after=...)``."""
     temporary_id_to_task = {
         task.attributes["collection_id"]: task
@@ -194,8 +189,7 @@ def _check_if_tasks_have_the_same_products(dag: nx.DiGraph, paths: list[Path]) -
         raise ResolvingDependenciesError(msg)
 
 
-@hookimpl
-def pytask_dag_log(report: DagReport) -> None:
+def _log_dag(report: DagReport) -> None:
     """Log errors which happened while resolving dependencies."""
     console.print()
     console.rule(
diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py
@@ -19,6 +19,7 @@
 from _pytask.config_utils import find_project_root_and_config
 from _pytask.config_utils import read_config
 from _pytask.console import console
+from _pytask.dag import create_dag
 from _pytask.exceptions import CollectionError
 from _pytask.exceptions import ConfigurationError
 from _pytask.exceptions import ResolvingDependenciesError
@@ -101,7 +102,7 @@ def dag(**raw_config: Any) -> int:
                 "can install with conda.",
             )
             session.hook.pytask_collect(session=session)
-            session.hook.pytask_dag(session=session)
+            session.dag = create_dag(session=session)
             dag = _refine_dag(session)
             _write_graph(dag, session.config["output_path"], session.config["layout"])
 
@@ -198,7 +199,7 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph:
                 "can install with conda.",
             )
             session.hook.pytask_collect(session=session)
-            session.hook.pytask_dag(session=session)
+            session.dag = create_dag(session=session)
             session.hook.pytask_unconfigure(session=session)
             dag = _refine_dag(session)
 
diff --git a/src/_pytask/data_catalog.py b/src/_pytask/data_catalog.py
@@ -99,7 +99,7 @@ def add(self, name: str, node: PNode | None = None) -> None:
             raise TypeError(msg)
 
         if node is None:
-            filename = str(hashlib.sha256(name.encode()).hexdigest())
+            filename = hashlib.sha256(name.encode()).hexdigest()
             if isinstance(self.default_node, PPathNode):
                 self.entries[name] = self.default_node(
                     name=name, path=self.path / f"{filename}.pkl"
diff --git a/src/_pytask/database_utils.py b/src/_pytask/database_utils.py
@@ -69,11 +69,10 @@ def update_states_in_database(session: Session, task_signature: str) -> None:
         _create_or_update_state(task_signature, node.signature, hash_)
 
 
-def has_node_changed(task: PTask, node: PTask | PNode) -> bool:
+def has_node_changed(task: PTask, node: PTask | PNode, state: str | None) -> bool:
     """Indicate whether a single dependency or product has changed."""
     # If node does not exist, we receive None.
-    node_state = node.state()
-    if node_state is None:
+    if state is None:
         return True
 
     with DatabaseSession() as session:
@@ -83,4 +82,4 @@ def has_node_changed(task: PTask, node: PTask | PNode) -> bool:
     if db_state is None:
         return True
 
-    return node_state != db_state.hash_
+    return state != db_state.hash_
diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py
@@ -57,7 +57,7 @@ def pytask_post_parse(config: dict[str, Any]) -> None:
 def pytask_execute(session: Session) -> None:
     """Execute tasks."""
     session.hook.pytask_execute_log_start(session=session)
-    session.scheduler = session.hook.pytask_execute_create_scheduler(session=session)
+    session.scheduler = TopologicalSorter.from_dag(session.dag)
     session.hook.pytask_execute_build(session=session)
     session.hook.pytask_execute_log_end(
         session=session, reports=session.execution_reports
@@ -73,12 +73,6 @@ def pytask_execute_log_start(session: Session) -> None:
     console.print()
 
 
-@hookimpl(trylast=True)
-def pytask_execute_create_scheduler(session: Session) -> TopologicalSorter:
-    """Create a scheduler based on topological sorting."""
-    return TopologicalSorter.from_dag(session.dag)
-
-
 @hookimpl
 def pytask_execute_build(session: Session) -> bool | None:
     """Execute tasks."""
@@ -140,7 +134,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
             node = dag.nodes[node_signature].get("task") or dag.nodes[
                 node_signature
             ].get("node")
-            if node_signature in predecessors and not node.state():
+            node_state = node.state()
+            if node_signature in predecessors and not node_state:
                 msg = f"{task.name!r} requires missing node {node.name!r}."
                 if IS_FILE_SYSTEM_CASE_SENSITIVE:
                     msg += (
@@ -149,7 +144,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
                     )
                 raise NodeNotFoundError(msg)
 
-            has_changed = has_node_changed(task=task, node=node)
+            has_changed = has_node_changed(task=task, node=node, state=node_state)
             if has_changed:
                 needs_to_be_executed = True
                 break
diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py
diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py
diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/test_collect.py b/tests/test_collect.py
diff --git a/tests/test_dag.py b/tests/test_dag.py
