From 472018f717cb7111db03ed4739138aa5e92b5b4a Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Mar 2024 18:12:26 +0100 Subject: [PATCH 1/3] Make universal-pathlib 0.2.2 an official dependency. --- docs/source/changes.md | 1 + docs/source/how_to_guides/remote_files.md | 7 +------ pyproject.toml | 2 +- src/_pytask/collect.py | 8 +------- tox.ini | 2 +- 5 files changed, 5 insertions(+), 15 deletions(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index 11401aa2..ab7a8e23 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -17,6 +17,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and relative to the config file. - {pull}`555` uses new-style hook wrappers and requires pluggy 1.3 for typing. - {pull}`557` fixes an issue with `@task(after=...)` in notebooks and terminals. +- {pull}`566` makes universal-pathlib an official dependency. ## 0.4.5 - 2024-01-09 diff --git a/docs/source/how_to_guides/remote_files.md b/docs/source/how_to_guides/remote_files.md index abd8967d..6125af9c 100644 --- a/docs/source/how_to_guides/remote_files.md +++ b/docs/source/how_to_guides/remote_files.md @@ -7,16 +7,11 @@ lots of use cases to deal with remote files. get started. So, some tasks reference remote files instead of local files. - You store the workflow results in remote storage to save and distribute them. -pytask uses [universal_pathlib](https://github.com/fsspec/universal_pathlib) to work +pytask uses [universal-pathlib](https://github.com/fsspec/universal_pathlib) to work with remote files. The package provides a {mod}`pathlib`-like interface, making it very easy to interact with files from an HTTP(S)-, Dropbox-, S3-, GCP-, Azure-based filesystem, and many more. -```{warning} -universal_pathlib does currently not support Python 3.12. To track progress, check the -[releases `>0.1.4`](https://github.com/fsspec/universal_pathlib/releases). -``` - ## HTTP(S)-based filesystem As an example for dealing with an HTTP(S)-based filesystem, we will download the iris diff --git a/pyproject.toml b/pyproject.toml index 934c73a2..19c17c80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "sqlalchemy>=2", 'tomli>=1; python_version < "3.11"', 'typing-extensions; python_version < "3.9"', + "universal-pathlib>=0.2.2", ] [project.readme] @@ -53,7 +54,6 @@ name = "Tobias Raabe" email = "raabe@posteo.de" [project.optional-dependencies] -all = ['universal-pathlib; python_version<"3.12"'] docs = [ "furo", "ipython", diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index bb0f2d95..bca6d070 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -44,13 +44,7 @@ from _pytask.task_utils import task as task_decorator from _pytask.typing import is_task_function from rich.text import Text - -try: - from upath import UPath -except ImportError: # pragma: no cover - - class UPath: # type: ignore[no-redef] - ... +from upath import UPath if TYPE_CHECKING: diff --git a/tox.ini b/tox.ini index fb088034..b3e06a80 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ passenv = CI package = editable [testenv:test] -extras = all, test +extras = test deps = pygraphviz;platform_system != "Windows" commands = From e70c0d137179cc8c609d7e07fad593499d960ef1 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Mar 2024 18:40:59 +0100 Subject: [PATCH 2/3] Simpify code and fix test. --- src/_pytask/nodes.py | 35 +++++++++++++++++++---------------- tests/test_execute.py | 1 - 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 26fd9e34..6d9f5d4e 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -20,6 +20,7 @@ from _pytask.typing import NoDefault from attrs import define from attrs import field +from upath import UPathStatResult if TYPE_CHECKING: @@ -179,14 +180,7 @@ def state(self) -> str | None: """ if self.path.exists(): - stat = self.path.stat() - if isinstance(stat, stat_result): - modification_time = self.path.stat().st_mtime - return hash_path(self.path, modification_time) - if isinstance(stat, dict): - return stat.get("ETag", "0") - msg = "Unknown stat object." - raise NotImplementedError(msg) + return _get_state(self.path) return None def load(self, is_product: bool = False) -> Path: # noqa: ARG002 @@ -322,14 +316,7 @@ def from_path(cls, path: Path) -> PickleNode: def state(self) -> str | None: if self.path.exists(): - stat = self.path.stat() - if isinstance(stat, stat_result): - modification_time = self.path.stat().st_mtime - return hash_path(self.path, modification_time) - if isinstance(stat, dict): - return stat.get("ETag", "0") - msg = "Unknown stat object." - raise NotImplementedError(msg) + return _get_state(self.path) return None def load(self, is_product: bool = False) -> Any: @@ -341,3 +328,19 @@ def load(self, is_product: bool = False) -> Any: def save(self, value: Any) -> None: with self.path.open("wb") as f: pickle.dump(value, f) + + +def _get_state(path: Path) -> str: + """Get state of a path. + + A simple function to handle local and remote files. + + """ + stat = path.stat() + if isinstance(stat, stat_result): + modification_time = path.stat().st_mtime + return hash_path(path, modification_time) + if isinstance(stat, UPathStatResult): + return stat.as_info().get("ETag", "0") + msg = "Unknown stat object." + raise NotImplementedError(msg) diff --git a/tests/test_execute.py b/tests/test_execute.py index d2060bae..db1834f7 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -918,7 +918,6 @@ def func(path): path.touch() assert tmp_path.joinpath("out.txt").exists() -@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Not supported in Python 3.12.") def test_with_http_path(runner, tmp_path): source = """ from upath import UPath From 1d78096f2a74731abc4649a03c6a70c6026c195d Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Mar 2024 18:43:33 +0100 Subject: [PATCH 3/3] fix. --- src/_pytask/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 6d9f5d4e..5cc3506b 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -20,7 +20,7 @@ from _pytask.typing import NoDefault from attrs import define from attrs import field -from upath import UPathStatResult +from upath._stat import UPathStatResult if TYPE_CHECKING: