Merge branch 'main' into use-macos-with-m1

Author: tobiasraabe

.github/workflows/main.yml

@@ -5,9 +5,6 @@ concurrency:
   group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
-env:
-  CONDA_EXE: mamba
-
 on:
   push:
     branches:
@@ -50,7 +47,7 @@ jobs:
         run: tox -e test -- -m "unit or (not integration and not end_to_end)" --cov=src --cov=tests --cov-report=xml -n auto
 
       - name: Upload unit test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           flags: unit
 
@@ -59,7 +56,7 @@ jobs:
         run: tox -e test -- -m integration --cov=src --cov=tests --cov-report=xml -n auto
 
       - name: Upload integration test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           flags: integration
 
@@ -68,6 +65,6 @@ jobs:
         run: tox -e test -- -m end_to_end --cov=src --cov=tests --cov-report=xml -n auto
 
       - name: Upload end_to_end test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           flags: end_to_end

.github/workflows/update-plugin-list.yml

@@ -37,7 +37,7 @@ jobs:
         run: python scripts/update_plugin_list.py
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38
+        uses: peter-evans/create-pull-request@a4f52f8033a6168103c2538976c07b467e8163bc
         with:
           commit-message: '[automated] Update plugin list'
           author: 'Tobias Raabe <[email protected]>'

.pre-commit-config.yaml

@@ -24,18 +24,6 @@ repos:
     -   id: python-check-mock-methods
     -   id: python-no-log-warn
     -   id: text-unicode-replacement-char
--   repo: https://github.com/asottile/reorder-python-imports
-    rev: v3.12.0
-    hooks:
-    -   id: reorder-python-imports
-        args: [--py38-plus, --add-import, 'from __future__ import annotations']
-        exclude: ^(docs_src/)
--   repo: https://github.com/asottile/reorder-python-imports
-    rev: v3.12.0
-    hooks:
-    -   id: reorder-python-imports
-        args: [--py38-plus]
-        files: ^(docs_src/)
 # -   repo: https://github.com/tox-dev/pyproject-fmt
 #     rev: 1.2.0
 #     hooks:
@@ -45,18 +33,18 @@ repos:
     hooks:
     -   id: sort-all
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.2.0
+    rev: v0.3.2
     hooks:
       - id: ruff-format
       - id: ruff
         args: [--unsafe-fixes]
 -   repo: https://github.com/dosisod/refurb
-    rev: v1.28.0
+    rev: v2.0.0
     hooks:
     -   id: refurb
         args: [--ignore, FURB126]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.9.0
     hooks:
     -   id: mypy
         additional_dependencies: [
@@ -90,14 +78,12 @@ repos:
         ]
         files: (docs/.)
 -   repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.7.1
+    rev: 1.8.4
     hooks:
-    -   id: nbqa-black
-    -   id: nbqa-isort
     -   id: nbqa-mypy
         args: [--ignore-missing-imports]
 -   repo: https://github.com/kynan/nbstripout
-    rev: 0.6.2
+    rev: 0.7.1
     hooks:
     -   id: nbstripout
         exclude: (docs)

docs/source/changes.md

@@ -16,6 +16,18 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
   passed via the command line are relative to CWD and paths in the configuration
   relative to the config file.
 - {pull}`555` uses new-style hook wrappers and requires pluggy 1.3 for typing.
+- {pull}`557` fixes an issue with `@task(after=...)` in notebooks and terminals.
+- {pull}`566` makes universal-pathlib an official dependency.
+- {pull}`568` restricts `task_files` to a list of patterns and raises a better error.
+- {pull}`569` removes the hooks related to the creation of the DAG.
+- {pull}`571` removes redundant calls to `PNode.state()` which causes a high penalty for
+  remote files.
+- {pull}`573` removes the `pytask_execute_create_scheduler` hook.
+
+## 0.4.6 - 2024-03-13
+
+- {pull}`576` fixes accidentally collecting `pytask.MarkGenerator` when using
+  `from pytask import mark`.
 
 ## 0.4.5 - 2024-01-09
 

docs/source/conf.py

@@ -4,6 +4,7 @@
 documentation: https://www.sphinx-doc.org/en/master/usage/configuration.html
 
 """
+
 from __future__ import annotations
 
 import inspect

docs/source/how_to_guides/remote_files.md

@@ -7,16 +7,11 @@ lots of use cases to deal with remote files.
   get started. So, some tasks reference remote files instead of local files.
 - You store the workflow results in remote storage to save and distribute them.
 
-pytask uses [universal_pathlib](https://github.com/fsspec/universal_pathlib) to work
+pytask uses [universal-pathlib](https://github.com/fsspec/universal_pathlib) to work
 with remote files. The package provides a {mod}`pathlib`-like interface, making it very
 easy to interact with files from an HTTP(S)-, Dropbox-, S3-, GCP-, Azure-based
 filesystem, and many more.
 
-```{warning}
-universal_pathlib does currently not support Python 3.12. To track progress, check the
-[releases `>0.1.4`](https://github.com/fsspec/universal_pathlib/releases).
-```
-
 ## HTTP(S)-based filesystem
 
 As an example for dealing with an HTTP(S)-based filesystem, we will download the iris

docs/source/reference_guides/configuration.md

@@ -242,7 +242,7 @@ strict_markers = true
 Change the pattern which identify task files.
 
 ```toml
-task_files = "task_*.py"  # default
+task_files = ["task_*.py"]  # default
 
 task_files = ["task_*.py", "tasks_*.py"]
 ```

docs/source/reference_guides/hookspecs.md

@@ -62,33 +62,13 @@ The following hooks traverse directories and collect tasks from files.
 .. autofunction:: pytask_collect_log
 ```
 
-## Resolving Dependencies
-
-The following hooks are designed to build a DAG from tasks and dependencies and check
-which files have changed and need to be re-run.
-
-```{warning}
-This step is still experimental and likely to change in the future. If you are planning
-to write a plugin which extends pytask in this dimension, please, start a discussion
-before writing a plugin. It may make your life easier if changes in pytask anticipate
-your plugin.
-```
-
-```{eval-rst}
-.. autofunction:: pytask_dag
-.. autofunction:: pytask_dag_create_dag
-.. autofunction:: pytask_dag_log
-
-```
-
 ## Execution
 
 The following hooks execute the tasks and log information on the result in the terminal.
 
 ```{eval-rst}
 .. autofunction:: pytask_execute
 .. autofunction:: pytask_execute_log_start
-.. autofunction:: pytask_execute_create_scheduler
 .. autofunction:: pytask_execute_build
 .. autofunction:: pytask_execute_task_protocol
 .. autofunction:: pytask_execute_task_log_start

docs/source/tutorials/defining_dependencies_products.md

@@ -72,7 +72,7 @@ task has finished, pytask will check whether the file exists.
 
 ````
 
-````{tab-item} prodouces
+````{tab-item} produces
 :sync: produces
 
 ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_products_produces.py
@@ -132,7 +132,7 @@ annotation are dependencies of the task.
 
 ````
 
-````{tab-item} prodouces
+````{tab-item} produces
 :sync: produces
 
 To specify that the task relies on the data set `data.pkl`, you can add the path to the
@@ -178,7 +178,7 @@ are assumed to point to a location relative to the task module.
 
 ````
 
-````{tab-item} prodouces
+````{tab-item} produces
 :sync: produces
 
 ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_relative_produces.py

docs/source/tutorials/making_tasks_persist.md

@@ -13,7 +13,7 @@ in the database such that the subsequent execution will skip the task successful
 
 ## When is this useful?
 
-- You ran a formatter like Black on the files in your project and want to prevent the
+- You ran a formatter like ruff on the files in your project and want to prevent the
   longest-running tasks from being rerun.
 - You extend a repetition of a task function but do not want to rerun all tasks.
 - You want to integrate a task that you have already run elsewhere. Copy over the

docs/source/tutorials/repeating_tasks_with_different_inputs.md

@@ -93,17 +93,17 @@ auto-generated ids are used.
 
 ### Auto-generated ids
 
-pytask construct ids by extending the task name with representations of the values used
+pytask constructs ids by extending the task name with representations of the values used
 for each iteration. Booleans, floats, integers, and strings enter the task id directly.
 For example, a task function that receives four arguments, `True`, `1.0`, `2`, and
-`"hello"`, one of each dtype, has the following id.
+`"hello"`, one of each data type, has the following id.
 
 ```
 task_data_preparation.py::task_create_random_data[True-1.0-2-hello]
 ```
 
-Arguments with other dtypes cannot be converted to strings and, thus, are replaced with
-a combination of the argument name and the iteration counter.
+Arguments with other data types cannot be converted to strings and, thus, are replaced
+with a combination of the argument name and the iteration counter.
 
 For example, the following function is parametrized with tuples.
 
@@ -136,7 +136,7 @@ For example, the following function is parametrized with tuples.
 
 Since the tuples are not converted to strings, the ids of the two tasks are
 
-```
+```text
 task_data_preparation.py::task_create_random_data[seed0]
 task_data_preparation.py::task_create_random_data[seed1]
 ```
@@ -177,7 +177,7 @@ a unique name for the iteration.
 
 produces these ids
 
-```
+```text
 task_data_preparation.py::task_create_random_data[first]
 task_data_preparation.py::task_create_random_data[second]
 ```

docs/source/tutorials/using_a_data_catalog.md

@@ -61,7 +61,7 @@ The following tabs show you how to use the data catalog given the interface you
 ````{tab-item} Python 3.10+
 :sync: python310plus
 
-Use `data_catalog["key"]` as an default argument to access the
+Use `data_catalog["data"]` as an default argument to access the
 {class}`~pytask.PickleNode` within the task. When you are done transforming your
 {class}`~pandas.DataFrame`, save it with {meth}`~pytask.PickleNode.save`.
 
@@ -74,7 +74,7 @@ Use `data_catalog["key"]` as an default argument to access the
 ````{tab-item} Python 3.8+
 :sync: python38plus
 
-Use `data_catalog["key"]` as an default argument to access the
+Use `data_catalog["data"]` as an default argument to access the
 {class}`~pytask.PickleNode` within the task. When you are done transforming your
 {class}`~pandas.DataFrame`, save it with {meth}`~pytask.PickleNode.save`.
 
@@ -87,7 +87,7 @@ Use `data_catalog["key"]` as an default argument to access the
 ````{tab-item} ​produces
 :sync: produces
 
-Use `data_catalog["key"]` as an default argument to access the
+Use `data_catalog["data"]` as an default argument to access the
 {class}`~pytask.PickleNode` within the task. When you are done transforming your
 {class}`~pandas.DataFrame`, save it with {meth}`~pytask.PickleNode.save`.
 
@@ -125,7 +125,7 @@ Following one of the interfaces gives you immediate access to the
 ````{tab-item} Python 3.10+
 :sync: python310plus
 
-Use `data_catalog["key"]` as an default argument to access the
+Use `data_catalog["data"]` as an default argument to access the
 {class}`~pytask.PickleNode` within the task. When you are done transforming your
 {class}`~pandas.DataFrame`, save it with {meth}`~pytask.PickleNode.save`.
 
@@ -138,7 +138,7 @@ Use `data_catalog["key"]` as an default argument to access the
 ````{tab-item} Python 3.8+
 :sync: python38plus
 
-Use `data_catalog["key"]` as an default argument to access the
+Use `data_catalog["data"]` as an default argument to access the
 {class}`~pytask.PickleNode` within the task. When you are done transforming your
 {class}`~pandas.DataFrame`, save it with {meth}`~pytask.PickleNode.save`.
 

docs_src/how_to_guides/bp_scaling_tasks_1.py

@@ -4,7 +4,6 @@
 from my_project.config import BLD
 from my_project.config import SRC
 
-
 DATA = {
     "data_0": {"subset": "subset_1"},
     "data_1": {"subset": "subset_2"},

docs_src/how_to_guides/bp_scaling_tasks_3.py

@@ -4,7 +4,6 @@
 from my_project.config import BLD
 from my_project.data_preparation.config import DATA
 
-
 _MODELS = ["linear_probability", "logistic_model", "decision_tree"]
 
 

docs_src/how_to_guides/bp_structure_of_task_files.py

@@ -26,9 +26,7 @@ def task_prepare_census_data(
     df.to_pickle(path_to_census)
 
 
-def _clean_data(df: pd.DataFrame) -> None:
-    ...
+def _clean_data(df: pd.DataFrame) -> None: ...
 
 
-def _create_new_variables(df: pd.DataFrame) -> None:
-    ...
+def _create_new_variables(df: pd.DataFrame) -> None: ...

docs_src/how_to_guides/interfaces/dependencies_annotation.py

@@ -4,5 +4,4 @@
 from pytask import PathNode
 
 
-def task_example(path: Annotated[Path, PathNode(path=Path("input.txt"))]) -> None:
-    ...
+def task_example(path: Annotated[Path, PathNode(path=Path("input.txt"))]) -> None: ...

docs_src/how_to_guides/interfaces/dependencies_default.py

@@ -1,5 +1,4 @@
 from pathlib import Path
 
 
-def task_example(path: Path = Path("input.txt")) -> None:
-    ...
+def task_example(path: Path = Path("input.txt")) -> None: ...

docs_src/how_to_guides/interfaces/dependencies_task_kwargs.py

@@ -4,5 +4,4 @@
 
 
 @task(kwargs={"path": Path("input.txt")})
-def task_example(path: Path) -> None:
-    ...
+def task_example(path: Path) -> None: ...

docs_src/how_to_guides/migrating_from_scripts_to_pytask_1.py

@@ -1,7 +1,6 @@
 # Content of task_data_management.py
 import pandas as pd
 
-
 df = pd.read_csv("data.csv")
 
 # Many operations.

docs_src/how_to_guides/remote_files/https.py

@@ -3,7 +3,6 @@
 
 from upath import UPath
 
-
 url = UPath("https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data")
 
 

docs_src/how_to_guides/using_task_returns_example_2_task.py

@@ -2,7 +2,6 @@
 
 from pytask import task
 
-
 func = lambda *x: "This is the content of the text file."
 
 

docs_src/how_to_guides/using_task_returns_example_3_task.py

@@ -2,7 +2,6 @@
 
 from pytask import task
 
-
 func = lambda *x: "This is the first content.", "This is the second content."