Initial commit.

Author: tobiasraabe

.github/workflows/main.yml

@@ -0,0 +1,34 @@
+name: main
+
+# Automatically cancel a previous run.
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  CONDA_EXE: mamba
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - '*'
+
+jobs:
+
+  run-type-checking:
+
+    name: Run tests for type-checking
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version-file: .python-version
+          allow-prereleases: true
+          cache: pip
+      - run: pip install tox-uv
+      - run: tox -e typing

.gitignore

@@ -0,0 +1,23 @@
+# Files
+.coverage
+coverage.*
+
+# Folders
+.idea
+.ipynb_checkpoints
+.tox
+.vscode
+_build
+__pycache__
+
+*.egg-info
+
+.pytask
+build
+dist
+src/project/_version.py
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.venv
+bld

.pre-commit-config.yaml

@@ -0,0 +1,80 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: check-added-large-files
+        args: ['--maxkb=25']
+    -   id: check-case-conflict
+    -   id: check-merge-conflict
+    -   id: check-vcs-permalinks
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: fix-byte-order-marker
+    -   id: mixed-line-ending
+    -   id: no-commit-to-branch
+        args: [--branch, main]
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+    -   id: python-check-blanket-noqa
+    -   id: python-check-mock-methods
+    -   id: python-no-eval
+    -   id: python-no-log-warn
+    -   id: python-use-type-annotations
+    -   id: text-unicode-replacement-char
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.4
+    hooks:
+      - id: ruff
+      - id: ruff-format
+-   repo: https://github.com/dosisod/refurb
+    rev: v2.0.0
+    hooks:
+    -   id: refurb
+-   repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.17
+    hooks:
+    -   id: mdformat
+        additional_dependencies: [
+            mdformat-myst,
+            mdformat-black,
+            mdformat-pyproject,
+        ]
+        files: (README\.md)
+# Conflicts with admonitions.
+# -   repo: https://github.com/executablebooks/mdformat
+#     rev: 0.7.17
+#     hooks:
+#     -   id: mdformat
+#         additional_dependencies: [
+#             mdformat-gfm,
+#             mdformat-black,
+#         ]
+#         args: [--wrap, "88"]
+- repo: https://github.com/codespell-project/codespell
+  rev: v2.2.6
+  hooks:
+    -   id: codespell
+# -   repo: https://github.com/pre-commit/mirrors-mypy
+#     rev: 'v1.9.0'
+#     hooks:
+#     -   id: mypy
+#         args: [
+#             --no-strict-optional,
+#             --ignore-missing-imports,
+#         ]
+#         additional_dependencies: [
+#             attrs,
+#             cloudpickle,
+#             loky,
+#             "git+https://github.com/pytask-dev/pytask.git@main",
+#             rich,
+#             types-click,
+#             types-setuptools,
+#         ]
+#         pass_filenames: false
+-   repo: meta
+    hooks:
+    -   id: check-hooks-apply
+    -   id: check-useless-excludes
+    # -   id: identity  # Prints all files passed to pre-commits. Debugging.

.python-version

@@ -0,0 +1 @@
+3.12.2

README.md

@@ -0,0 +1,4 @@
+# coiled-project
+
+This project allows testing all possible interactions between pytask, pytask-parallel
+and coiled.

pyproject.toml

@@ -0,0 +1,69 @@
+[project]
+name = "project"
+version = "0.1.0"
+description = "A project to test coiled."
+authors = [
+    { name = "Tobias Raabe", email = "[email protected]" }
+]
+dependencies = [
+    "pytask>=0.4.5",
+    "pytask-parallel[coiled]>=0.4.1",
+    "s3fs>=2024.3.1",
+]
+readme = "README.md"
+requires-python = ">= 3.8"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.mypy]
+files = ["src"]
+check_untyped_defs = true
+disallow_any_generics = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+disallow_untyped_defs = false
+ignore_errors = true
+
+[tool.rye]
+managed = true
+dev-dependencies = []
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.ruff]
+target-version = "py38"
+fix = true
+unsafe-fixes = true
+
+[tool.ruff.lint]
+extend-ignore = [
+    "ANN101",  # type annotating self
+    "ANN102",  # type annotating cls
+    "ANN401",  # flake8-annotate typing.Any
+    "COM812",  # Comply with ruff-format.
+    "ISC001",  # Comply with ruff-format.
+    "D",
+]
+select = ["ALL"]
+
+[tool.ruff.lint.per-file-ignores]
+"src/*" = ["S101"]
+"tests/*" = ["D", "ANN", "PLR2004", "S101"]
+
+[tool.ruff.lint.isort]
+force-single-line = true
+
+[tool.ruff.lint.pydocstyle]
+convention = "numpy"
+
+[tool.pytask.ini_options]

src/project/__init__.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import coiled
+from pytask import DataCatalog
+from pytask_parallel import ParallelBackend
+from pytask_parallel import registry
+
+if TYPE_CHECKING:
+    from concurrent.futures import Executor
+
+SRC = Path(__file__).parent
+BLD = SRC.parent.parent / "bld"
+
+
+def _build_client(n_workers: int) -> Executor:
+    return (
+        coiled.Cluster(n_workers=n_workers, name="coiled-project")
+        .get_client()
+        .get_executor()
+    )
+
+
+registry.register_parallel_backend(ParallelBackend.CUSTOM, _build_client, remote=True)
+
+
+data_catalog = DataCatalog()

src/project/config.py

@@ -0,0 +1,27 @@
+from typing import cast
+
+from pytask import PickleNode
+from pytask import Product
+from typing_extensions import Annotated
+
+from project.config import BLD
+from project.config import data_catalog
+
+
+def task_create_data_catalog_node(
+    node: Annotated[PickleNode, Product] = cast(PickleNode, data_catalog["first"]),  # noqa: B008
+) -> None:
+    node.save("Hello,")
+
+
+def task_use_data_catalog_return(
+    text: Annotated[str, data_catalog["first"]],
+) -> Annotated[str, data_catalog["second"]]:
+    return text + " World!"
+
+
+def task_data_catalog_check(
+    text: Annotated[str, data_catalog["second"]],
+) -> Annotated[str, BLD / "data_catalog" / "output.txt"]:
+    assert text == "Hello, World!"
+    return text

src/project/task_data_catalog.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+from pytask import Product
+from typing_extensions import Annotated
+
+from project.config import BLD
+
+
+def task_create_local_path_node(
+    path: Annotated[Path, Product] = BLD / "local_path" / "first.txt",
+) -> None:
+    path.write_text("Hello,")
+
+
+def task_use_local_path_return(
+    path: Annotated[Path, BLD / "local_path" / "first.txt"],
+) -> Annotated[str, BLD / "local_path" / "second.txt"]:
+    return path.read_text() + " World!"
+
+
+def task_local_file_check(path: Path = BLD / "local_path" / "second.txt") -> None:
+    assert path.read_text() == "Hello, World!"

src/project/task_local_path.py

@@ -0,0 +1,25 @@
+from pytask import Product
+from pytask import PythonNode
+from typing_extensions import Annotated
+
+from project.config import BLD
+
+first_part = PythonNode()
+
+
+def task_first(node: Annotated[PythonNode, first_part, Product]) -> None:
+    node.save("Hello ")
+
+
+full_text = PythonNode()
+
+
+def task_second(first_part: Annotated[str, first_part]) -> Annotated[str, full_text]:
+    return first_part + "World!"
+
+
+def task_third(
+    full_text: Annotated[str, full_text],
+) -> Annotated[str, BLD / "python_nodes" / "output.txt"]:
+    assert full_text == "Hello World!"
+    return full_text

src/project/task_python_nodes.py

@@ -0,0 +1,24 @@
+[tox]
+requires = tox>=4
+envlist = test
+
+[testenv]
+package = editable
+
+[testenv:typing]
+deps =
+    mypy
+    git+https://github.com/pytask-dev/pytask.git@main
+    git+https://github.com/pytask-dev/pytask-parallel.git@main
+extras =
+    dask
+    coiled
+commands = mypy
+
+[testenv:test]
+extras = test
+deps =
+    git+https://github.com/pytask-dev/pytask.git@main
+    git+https://github.com/pytask-dev/pytask-parallel.git@main
+commands =
+    pytest {posargs}