Release v0.0.5 and make it work with pytask v0.0.9. (#5)

Author: tobiasraabe

.conda/meta.yaml

@@ -20,11 +20,12 @@ requirements:
 
   run:
     - python >=3.6
-    - pytask >=0.0.7
+    - pytask >=0.0.9
 
 test:
   requires:
     - pytest
+    - pytask-parallel >=0.0.4
     - r-base
   source_files:
     - tox.ini
@@ -34,6 +35,7 @@ test:
     - pytask --help
     - pytask markers
     - pytask clean
+    - pytask collect
 
     - pytest tests
 

CHANGES.rst

@@ -6,6 +6,12 @@ chronological order. Releases follow `semantic versioning <https://semver.org/>`
 all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask-r>`_.
 
 
+0.0.5 - 2020-10-30
+------------------
+
+- :gh:`5` makes pytask-r work with pytask v0.0.9.
+
+
 0.0.4 - 2020-10-14
 ------------------
 

README.rst

@@ -72,8 +72,18 @@ Here is an example where you want to run ``script.r``.
         pass
 
 Note that, you need to apply the ``@pytask.mark.r`` marker so that pytask-r handles the
-task. The executable script must be the first dependency. Other dependencies can be
-added after that.
+task.
+
+If you are wondering why the function body is empty, know that pytask-r replaces the
+body with a predefined internal function. See the section on implementation details for
+more information.
+
+
+Multiple dependencies and products
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+What happens if a task has more dependencies? Using a list, the R script which should be
+executed must be found in the first position of the list.
 
 .. code-block:: python
 
@@ -83,9 +93,31 @@ added after that.
     def task_run_r_script():
         pass
 
-If you are wondering why the function body is empty, know that pytask-r replaces the
-body with a predefined internal function. See the section on implementation details for
-more information.
+If you use a dictionary to pass dependencies to the task, pytask-r will, first, look
+for a ``"source"`` key in the dictionary and, secondly, under the key ``0``.
+
+.. code-block:: python
+
+    @pytask.mark.depends_on({"source": "script.r", "input": "input.rds"})
+    def task_run_r_script():
+        pass
+
+
+    # or
+
+
+    @pytask.mark.depends_on({0: "script.r", "input": "input.rds"})
+    def task_run_r_script():
+        pass
+
+
+    # or two decorators for the function, if you do not assign a name to the input.
+
+
+    @pytask.mark.depends_on({"source": "script.r"})
+    @pytask.mark.depends_on("input.rds")
+    def task_run_r_script():
+        pass
 
 
 Command Line Arguments
@@ -138,12 +170,26 @@ include the ``@pytask.mark.r`` decorator in the parametrization just like with
     @pytask.mark.depends_on("script.r")
     @pytask.mark.parametrize(
         "produces, r",
-        [("output_1.rds", ["--vanilla", 1]), ("output_2.rds", ["--vanilla", 2])],
+        [
+            ("output_1.rds", (["--vanilla", "1"],)),
+            ("output_2.rds", (["--vanilla", "2"],)),
+        ],
     )
     def task_execute_r_script():
         pass
 
 
+Configuration
+-------------
+
+If you want to change the name of the key which identifies the R script, change the
+following default configuration in your pytask configuration file.
+
+.. code-block:: ini
+
+    r_source_key = source
+
+
 Implementation Details
 ----------------------
 

environment.yml

@@ -13,7 +13,8 @@ dependencies:
   - conda-verify
 
   # Package dependencies
-  - pytask >= 0.0.7
+  - pytask >=0.0.9
+  - pytask-parallel >=0.0.4
   - r-base
 
   # Misc

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.4
+current_version = 0.0.5
 parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))(\-?((dev)?(?P<dev>\d+))?)
 serialize = 
 	{major}.{minor}.{patch}dev{dev}

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name="pytask-r",
-    version="0.0.4",
+    version="0.0.5",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     entry_points={"pytask": ["pytask_r = pytask_r.plugin"]},

src/pytask_r/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.4"
+__version__ = "0.0.5"

src/pytask_r/collect.py

@@ -2,6 +2,7 @@
 import copy
 import functools
 import subprocess
+from pathlib import Path
 from typing import Iterable
 from typing import Optional
 from typing import Union
@@ -12,7 +13,6 @@
 from _pytask.nodes import FilePathNode
 from _pytask.nodes import PythonFunctionTask
 from _pytask.parametrize import _copy_func
-from _pytask.shared import to_list
 
 
 def r(options: Optional[Union[str, Iterable[str]]] = None):
@@ -31,10 +31,9 @@ def r(options: Optional[Union[str, Iterable[str]]] = None):
     return options
 
 
-def run_r_script(depends_on, r):
+def run_r_script(r):
     """Run an R script."""
-    script = to_list(depends_on)[0]
-    subprocess.run(["Rscript", script.as_posix(), *r], check=True)
+    subprocess.run(r, check=True)
 
 
 @hookimpl
@@ -50,32 +49,37 @@ def pytask_collect_task(session, path, name, obj):
         task = PythonFunctionTask.from_path_name_function_session(
             path, name, obj, session
         )
+
+        return task
+
+
+@hookimpl
+def pytask_collect_task_teardown(session, task):
+    """Perform some checks."""
+    if get_specific_markers_from_task(task, "r"):
+        source = _get_node_from_dictionary(task.depends_on, "source")
+        if isinstance(source, FilePathNode) and source.value.suffix not in [".r", ".R"]:
+            raise ValueError(
+                "The first dependency of an R task must be the executable script."
+            )
+
         r_function = _copy_func(run_r_script)
         r_function.pytaskmark = copy.deepcopy(task.function.pytaskmark)
 
         merged_marks = _merge_all_markers(task)
         args = r(*merged_marks.args, **merged_marks.kwargs)
-        r_function = functools.partial(r_function, r=args)
+        options = _prepare_cmd_options(session, task, args)
+        r_function = functools.partial(r_function, r=options)
 
         task.function = r_function
 
-        return task
-
 
-@hookimpl
-def pytask_collect_task_teardown(task):
-    """Perform some checks.
-
-    Remove is task is none check with pytask 0.0.9.
-
-    """
-    if task is not None and get_specific_markers_from_task(task, "r"):
-        if isinstance(task.depends_on[0], FilePathNode) and task.depends_on[
-            0
-        ].value.suffix not in [".r", ".R"]:
-            raise ValueError(
-                "The first dependency of an R task must be the executable script."
-            )
+def _get_node_from_dictionary(obj, key, fallback=0):
+    if isinstance(obj, Path):
+        pass
+    elif isinstance(obj, dict):
+        obj = obj.get(key) or obj.get(fallback)
+    return obj
 
 
 def _merge_all_markers(task):
@@ -85,3 +89,14 @@ def _merge_all_markers(task):
     for mark_ in r_marks[1:]:
         mark = mark.combined_with(mark_)
     return mark
+
+
+def _prepare_cmd_options(session, task, args):
+    """Prepare the command line arguments to execute the do-file.
+
+    The last entry changes the name of the log file. We take the task id as a name which
+    is unique and does not cause any errors when parallelizing the execution.
+
+    """
+    source = _get_node_from_dictionary(task.depends_on, session.config["r_source_key"])
+    return ["Rscript", source.value.as_posix(), *args]

src/pytask_r/config.py

@@ -3,6 +3,7 @@
 
 
 @hookimpl
-def pytask_parse_config(config):
+def pytask_parse_config(config, config_from_file):
     """Register the r marker."""
     config["markers"]["r"] = "Tasks which are executed with Rscript."
+    config["r_source_key"] = config_from_file.get("r_source_key", "source")

tests/conftest.py

@@ -1,8 +1,13 @@
 import shutil
 
 import pytest
-
+from click.testing import CliRunner
 
 needs_rscript = pytest.mark.skipif(
     shutil.which("Rscript") is None, reason="R with Rscript needs to be installed."
 )
+
+
+@pytest.fixture()
+def runner():
+    return CliRunner()

tests/test_collect.py

@@ -4,7 +4,9 @@
 import pytest
 from _pytask.mark import Mark
 from _pytask.nodes import FilePathNode
+from pytask_r.collect import _get_node_from_dictionary
 from pytask_r.collect import _merge_all_markers
+from pytask_r.collect import _prepare_cmd_options
 from pytask_r.collect import pytask_collect_task
 from pytask_r.collect import pytask_collect_task_teardown
 from pytask_r.collect import r
@@ -82,12 +84,61 @@ def test_pytask_collect_task(name, expected):
         (["input.rds", "script.R"], ["any_out.rds"], pytest.raises(ValueError)),
     ],
 )
-def test_pytask_collect_task_teardown(depends_on, produces, expectation):
+@pytest.mark.parametrize("r_source_key", ["source", "script"])
+def test_pytask_collect_task_teardown(depends_on, produces, expectation, r_source_key):
+    session = DummyClass()
+    session.config = {"r_source_key": r_source_key}
+
     task = DummyClass()
-    task.depends_on = [FilePathNode(n.split(".")[0], Path(n)) for n in depends_on]
-    task.produces = [FilePathNode(n.split(".")[0], Path(n)) for n in produces]
+    task.depends_on = {
+        i: FilePathNode(n.split(".")[0], Path(n)) for i, n in enumerate(depends_on)
+    }
+    task.produces = {
+        i: FilePathNode(n.split(".")[0], Path(n)) for i, n in enumerate(produces)
+    }
     task.markers = [Mark("r", (), {})]
     task.function = task_dummy
+    task.function.pytaskmark = task.markers
 
     with expectation:
-        pytask_collect_task_teardown(task)
+        pytask_collect_task_teardown(session, task)
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    "obj, key, expected",
+    [
+        (1, "asds", 1),
+        (1, None, 1),
+        ({"a": 1}, "a", 1),
+        ({0: 1}, "a", 1),
+    ],
+)
+def test_get_node_from_dictionary(obj, key, expected):
+    result = _get_node_from_dictionary(obj, key)
+    assert result == expected
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    "args",
+    [
+        [],
+        ["a"],
+        ["a", "b"],
+    ],
+)
+@pytest.mark.parametrize("r_source_key", ["source", "script"])
+def test_prepare_cmd_options(args, r_source_key):
+    session = DummyClass()
+    session.config = {"r_source_key": r_source_key}
+
+    node = DummyClass()
+    node.value = Path("script.r")
+    task = DummyClass()
+    task.depends_on = {r_source_key: node}
+    task.name = "task"
+
+    result = _prepare_cmd_options(session, task, args)
+
+    assert result == ["Rscript", "script.r", *args]