Merge branch 'main' into add-datacatalog-name

tobiasraabe · tobiasraabe · commit adf9376760db · 2024-12-23T22:50:53.000+01:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -62,7 +62,7 @@ jobs:
         run: tox -e test -- -m "unit or (not integration and not end_to_end)" --cov=src --cov=tests --cov-report=xml -n auto
 
       - name: Upload unit test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           flags: unit
 
@@ -71,7 +71,7 @@ jobs:
         run: tox -e test -- -m integration --cov=src --cov=tests --cov-report=xml -n auto
 
       - name: Upload integration test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           flags: integration
 
@@ -80,6 +80,6 @@ jobs:
         run: tox -e test -- -m end_to_end --cov=src --cov=tests --cov-report=xml -n auto
 
       - name: Upload end_to_end test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           flags: end_to_end
diff --git a/.gitignore b/.gitignore
@@ -27,3 +27,4 @@ tests/test_jupyter/*.txt
 .ruff_cache
 .venv
 docs/jupyter_execute
+uv.lock
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -24,12 +24,8 @@ repos:
     -   id: python-check-mock-methods
     -   id: python-no-log-warn
     -   id: text-unicode-replacement-char
--   repo: https://github.com/aio-libs/sort-all
-    rev: v1.2.0
-    hooks:
-    -   id: sort-all
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.8.4
     hooks:
       - id: ruff-format
       - id: ruff
@@ -38,32 +34,32 @@ repos:
     hooks:
     -   id: refurb
 -   repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.18
+    rev: 0.7.21
     hooks:
     -   id: mdformat
         additional_dependencies: [
             mdformat-gfm,
-            mdformat-black,
-            mdformat-pyproject,
+            mdformat-ruff,
+            mdformat-pyproject>=0.0.2,
         ]
         files: (README\.md)
 -   repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.18
+    rev: 0.7.21
     hooks:
     -   id: mdformat
         additional_dependencies: [
             mdformat-myst,
-            mdformat-black,
-            mdformat-pyproject,
+            mdformat-ruff,
+            mdformat-pyproject>=0.0.2,
         ]
         files: (docs/.)
 -   repo: https://github.com/kynan/nbstripout
-    rev: 0.7.1
+    rev: 0.8.1
     hooks:
     -   id: nbstripout
         exclude: (docs)
 -   repo: https://github.com/crate-ci/typos
-    rev: v1.26.8
+    rev: v1.28.4
     hooks:
     -   id: typos
         exclude: (\.ipynb)
diff --git a/docs/source/changes.md b/docs/source/changes.md
@@ -5,12 +5,16 @@ chronological order. Releases follow [semantic versioning](https://semver.org/)
 releases are available on [PyPI](https://pypi.org/project/pytask) and
 [Anaconda.org](https://anaconda.org/conda-forge/pytask).
 
-## 0.5.2 - 2024-09-15
+## 0.5.2 - 2024-12-19
 
 - {pull}`633` adds support for Python 3.13 and drops support for 3.8.
 - {pull}`640` stops the live display when an exception happened during the execution.
 - {pull}`646` adds a `.gitignore` to the `.pytask/` folder to exclude it from version
   control.
+- {pull}`656` fixes the return type of the hash function for {class}`PythonNode`s.
+  Thanks to {user}`axtimhaus` for reporting the issue.
+- {pull}`657` documents `pipefunc`, another tool for executing graphs consisting out of
+  functions.
 
 ## 0.5.1 - 2024-07-20
 
diff --git a/docs/source/explanations/comparison_to_other_tools.md b/docs/source/explanations/comparison_to_other_tools.md
@@ -126,3 +126,8 @@ General
 ## [zenml](https://github.com/zenml-io/zenml)
 
 ## [flyte](https://github.com/flyteorg/flyte)
+
+## [pipefunc](https://github.com/pipefunc/pipefunc)
+
+A tool for executing graphs made out of functions. More focused on computational
+compared to workflow graphs.
diff --git a/docs/source/how_to_guides/hashing_inputs_of_tasks.md b/docs/source/how_to_guides/hashing_inputs_of_tasks.md
@@ -44,7 +44,7 @@ the `__hash__()` method of the object.
 Some objects like {class}`tuple` and {class}`typing.NamedTuple` are hashable and return
 correct hashes by default.
 
-```python
+```pycon
 >>> hash((1, 2))
 -3550055125485641917
 ```
@@ -54,7 +54,7 @@ from interpreter session to interpreter session for security reasons (see
 {meth}`object.__hash__` for more information). pytask will hash them using the
 {mod}`hashlib` module to create a stable hash.
 
-```python
+```pycon
 >>> from pytask import PythonNode
 >>> node = PythonNode(value="Hello, World!", hash=True)
 >>> node.state()
@@ -72,7 +72,8 @@ $ pip install deepdiff
 $ conda install deepdiff
 ```
 
-Then, create the hash function and pass it to the node.
+Then, create the hash function and pass it to the node. Make sure it returns either an
+integer or a string.
 
 `````{tab-set}
 
diff --git a/docs/source/how_to_guides/provisional_nodes_and_task_generators.md b/docs/source/how_to_guides/provisional_nodes_and_task_generators.md
@@ -89,6 +89,6 @@ content to a `.txt` file.
 ```
 
 ```{important}
-The generated tasks need to be decoratored with {func}`@task <pytask.task>` to be
+The generated tasks need to be decorated with {func}`@task <pytask.task>` to be
 collected.
 ```
diff --git a/docs/source/how_to_guides/the_data_catalog.md b/docs/source/how_to_guides/the_data_catalog.md
@@ -122,7 +122,7 @@ for model_name in MODEL_NAMES:
 
         @task
         def fit_model(
-            path: Path = Path("...", data_name)
+            path: Path = Path("...", data_name),
         ) -> Annotated[
             Any, nested_data_catalogs[model_name][data_name]["fitted_model"]
         ]:
diff --git a/docs/source/how_to_guides/writing_custom_nodes.md b/docs/source/how_to_guides/writing_custom_nodes.md
@@ -142,9 +142,9 @@ databases. [^kedro]
 
 ## References
 
-[^structural-subtyping]: Structural subtyping is similar to ABCs an approach in Python to enforce interfaces, but
-    it can be considered more pythonic since it is closer to duck typing. Hynek Schlawack
-    wrote a comprehensive
+[^structural-subtyping]: Structural subtyping is similar to ABCs an approach in Python to enforce interfaces,
+    but it can be considered more pythonic since it is closer to duck typing. Hynek
+    Schlawack wrote a comprehensive
     [guide on subclassing](https://hynek.me/articles/python-subclassing-redux/) that
     features protocols under "Type 2". Glyph wrote an introduction to protocols called
     [I want a new duck](https://glyph.twistedmatrix.com/2020/07/new-duck.html).
diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md
@@ -257,8 +257,7 @@ Applied to the tasks from before, we could have written `task_plot_data` as
 
 ```python
 @task(after=task_create_random_data)
-def task_plot_data(...):
-    ...
+def task_plot_data(): ...
 ```
 
 You can also pass a list of task functions.
@@ -268,8 +267,7 @@ tasks. Here, we can pass the function name or a significant part of the function
 
 ```python
 @task(after="random_data")
-def task_plot_data(...):
-    ...
+def task_plot_data(): ...
 ```
 
 You will learn more about expressions in {doc}`selecting_tasks`.
diff --git a/docs_src/how_to_guides/hashing_inputs_of_tasks_example_3_py310.py b/docs_src/how_to_guides/hashing_inputs_of_tasks_example_3_py310.py
@@ -9,7 +9,7 @@
 from pytask import PythonNode
 
 
-def calculate_hash(x: Any) -> str:
+def calculate_hash(x: Any) -> int | str:
     return DeepHash(x)[x]
 
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -31,7 +31,7 @@ dependencies = [
     "rich",
     "sqlalchemy>=2",
     'tomli>=1; python_version < "3.11"',
-    'typing-extensions; python_version < "3.9"',
+    'typing-extensions; python_version < "3.11"',
     "universal-pathlib>=0.2.2",
 ]
 
@@ -86,7 +86,7 @@ pytask = "pytask:cli"
 
 [tool.uv]
 dev-dependencies = [
-    "tox-uv>=1.7.0", "pygraphviz"
+    "tox-uv>=1.7.0", "pygraphviz;platform_system=='Linux'",
 ]
 
 [build-system]
@@ -119,8 +119,6 @@ extend-include = ["*.ipynb"]
 [tool.ruff.lint]
 select = ["ALL"]
 ignore = [
-    "ANN101",
-    "ANN102",
     "ANN401", # flake8-annotate typing.Any
     "COM812", # Comply with ruff-format.
     "ISC001", # Comply with ruff-format.
@@ -144,7 +142,7 @@ ignore = [
 "scripts/*" = ["D", "INP001"]
 "docs/source/conf.py" = ["D401", "INP001"]
 "docs_src/*" = ["ARG001", "D", "INP001", "S301"]
-"docs_src/*/*.py" = ["FA100", "FA102", "PLR2004", "TCH"]
+"docs_src/*/*.py" = ["FA100", "FA102", "PLR2004", "TC"]
 "docs/source/how_to_guides/functional_interface*" = [
     "B018",
     "D",
diff --git a/scripts/update_plugin_list.py b/scripts/update_plugin_list.py
@@ -78,7 +78,7 @@
 )
 
 
-_EXCLUDED_PACKAGES = ["pytask-io", "pytask-list"]
+_EXCLUDED_PACKAGES = ["pytask-io", "pytask-list", "pytask-queue"]
 
 
 def _escape_rst(text: str) -> str:
diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py
@@ -328,7 +328,7 @@ def repr(self, class_name: str) -> str:
         return "<{} {} _old={} _state={!r} tmpfile={!r}>".format(
             class_name,
             self.name,
-            hasattr(self, "_old") and repr(self._old) or "<UNSET>",
+            (hasattr(self, "_old") and repr(self._old)) or "<UNSET>",
             self._state,
             self.tmpfile,
         )
@@ -337,7 +337,7 @@ def __repr__(self) -> str:
         return "<{} {} _old={} _state={!r} tmpfile={!r}>".format(
             self.__class__.__name__,
             self.name,
-            hasattr(self, "_old") and repr(self._old) or "<UNSET>",
+            (hasattr(self, "_old") and repr(self._old)) or "<UNSET>",
             self._state,
             self.tmpfile,
         )
diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py
@@ -150,7 +150,7 @@ def _import_pdb_cls(
                 __import__(modname)
                 mod = sys.modules[modname]
 
-                # Handle --pdbcls=pdb:pdb.Pdb (useful e.g. with pdbpp).
+                # Handle --pdbcls=pdb:pdb.Pdb (useful e.g. with pdbpp or pdbp).
                 parts = classname.split(".")
                 pdb_cls = getattr(mod, parts[0])
                 for part in parts[1:]:
diff --git a/src/_pytask/live.py b/src/_pytask/live.py
@@ -114,7 +114,7 @@ def start(self) -> None:
             self._live.start()
         except LiveError:
             msg = (
-                "pytask tried to launch a second live display which is impossible. the "
+                "pytask tried to launch a second live display which is impossible. The "
                 "issue occurs when you use pytask on the command line on a task module "
                 "that uses the programmatic interface of pytask at the same time. "
                 "Use either the command line or the programmatic interface."
diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py
@@ -36,8 +36,8 @@
 
 
 __all__ = [
-    "Expression",
     "MARK_GEN",
+    "Expression",
     "Mark",
     "MarkDecorator",
     "MarkGenerator",
diff --git a/src/_pytask/mark/expression.py b/src/_pytask/mark/expression.py
@@ -81,7 +81,7 @@ def __str__(self) -> str:
 
 
 class Scanner:
-    __slots__ = ("tokens", "current")
+    __slots__ = ("current", "tokens")
 
     def __init__(self, input_: str) -> None:
         self.tokens = self.lex(input_)
diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py
@@ -7,7 +7,7 @@
 import pickle
 from contextlib import suppress
 from os import stat_result
-from pathlib import Path  # noqa: TCH003
+from pathlib import Path  # noqa: TC003
 from typing import TYPE_CHECKING
 from typing import Any
 from typing import Callable
@@ -217,7 +217,7 @@ class PythonNode(PNode):
         Whether the value should be hashed to determine the state. Use ``True`` for
         objects that are hashable like strings and tuples. For dictionaries and other
         non-hashable objects, you need to provide a function that can hash these
-        objects.
+        objects. The function should return either an integer or a string.
     node_info
         The infos acquired while collecting the node.
 
@@ -236,7 +236,7 @@ class PythonNode(PNode):
 
     name: str = ""
     value: Any | NoDefault = no_default
-    hash: bool | Callable[[Any], bool] = False
+    hash: bool | Callable[[Any], int | str] = False
     node_info: NodeInfo | None = None
     attributes: dict[Any, Any] = field(factory=dict)
 
@@ -271,7 +271,8 @@ def state(self) -> str | None:
         If ``hash = False``, the function returns ``"0"``, a constant hash value, so the
         :class:`PythonNode` is ignored when checking for a changed state of the task.
 
-        If ``hash`` is a callable, then use this function to calculate a hash.
+        If ``hash`` is a callable, then use this function to calculate a hash expecting
+        an integer or string.
 
         If ``hash = True``, the builtin ``hash()`` function (`link
         <https://docs.python.org/3.11/library/functions.html?highlight=hash#hash>`_) is
diff --git a/src/_pytask/tree_util.py b/src/_pytask/tree_util.py
@@ -14,8 +14,8 @@
 from optree import tree_structure as _optree_tree_structure
 
 __all__ = [
-    "PyTree",
     "TREE_UTIL_LIB_DIRECTORY",
+    "PyTree",
     "tree_flatten_with_path",
     "tree_leaves",
     "tree_map",
diff --git a/tests/test_collect_utils.py b/tests/test_collect_utils.py
@@ -5,7 +5,7 @@
 import pytest
 
 from _pytask.collect_utils import _find_args_with_product_annotation
-from pytask import Product  # noqa: TCH001
+from pytask import Product
 
 
 @pytest.mark.unit
diff --git a/tests/test_debugging.py b/tests/test_debugging.py
@@ -329,8 +329,8 @@ def task_1():
 @pytest.mark.skipif(not IS_PEXPECT_INSTALLED, reason="pexpect is not installed.")
 @pytest.mark.skipif(sys.platform == "win32", reason="pexpect cannot spawn on Windows.")
 def test_pdb_with_injected_do_debug(tmp_path):
-    """Simulates pdbpp, which injects Pdb into do_debug, and uses self.__class__ in
-    do_continue."""
+    """Simulates pdbpp or pdbp, which injects Pdb into do_debug, and uses self.__class__
+    in do_continue."""
     source = """
     import pdb
 
diff --git a/uv.lock b/uv.lock
