Limit number of items in the execution table during the execution. (#151)

Author: tobiasraabe

Diff for: docs/source/changes.rst

@@ -14,6 +14,9 @@ all releases are available on `PyPI <https://pypi.org/project/pytask>`_ and
   table for the default verbosity level of 1. They are displayed at 2.
 - :gh:`144` adds tryceratops to the pre-commit hooks for catching issues with
   exceptions.
+- :gh:`150` adds a limit on the number of items displayed in the execution table which
+  is also configurable with ``--n-entries-in-table`` on the cli and
+  ``n_entries_in_table`` in the configuration file.
 - :gh:`152` makes the duration of the execution readable by humans by separating it into
   days, hours, minutes and seconds.
 

Diff for: docs/source/reference_guides/configuration.rst

@@ -97,6 +97,23 @@ The options
             wip: Work-in-progress. These are tasks which I am currently working on.
 
 
+.. confval:: n_entries_in_table
+
+    You can limit the number of entries displayed in the live table during the execution
+    to make it more clear. Use either ``all`` or an integer greater or equal to one. On
+    the command line use
+
+    .. code-block:: console
+
+        $ pytask build --n-entries-in-table 10
+
+    and in the configuration use
+
+    .. code-block:: ini
+
+        n_entries_in_table = all  # default 15
+
+
 .. confval:: paths
 
     If you want to collect tasks from specific paths without passing the names via the

Diff for: src/_pytask/live.py

@@ -1,22 +1,71 @@
 from pathlib import Path
+from typing import Union
 
 import attr
+import click
 from _pytask.config import hookimpl
 from _pytask.console import console
+from _pytask.shared import get_first_non_none_value
 from _pytask.shared import reduce_node_name
 from rich.live import Live
 from rich.status import Status
 from rich.table import Table
 from rich.text import Text
 
 
+@hookimpl
+def pytask_extend_command_line_interface(cli):
+    """Extend command line interface."""
+    additional_parameters = [
+        click.Option(
+            ["--n-entries-in-table"],
+            default=None,
+            help="How many entries to display in the table during the execution. "
+            "Tasks which are running are always displayed.  [default: 15]",
+        ),
+    ]
+    cli.commands["build"].params.extend(additional_parameters)
+
+
+@hookimpl
+def pytask_parse_config(config, config_from_cli, config_from_file):
+    config["n_entries_in_table"] = get_first_non_none_value(
+        config_from_cli,
+        config_from_file,
+        key="n_entries_in_table",
+        default=15,
+        callback=_parse_n_entries_in_table,
+    )
+
+
+def _parse_n_entries_in_table(value: Union[int, str, None]) -> int:
+    if value in ["none", "None", None, ""]:
+        out = None
+    elif isinstance(value, int) and value >= 1:
+        out = value
+    elif isinstance(value, str) and value.isdigit() and int(value) >= 1:
+        out = int(value)
+    elif value == "all":
+        out = 1_000_000
+    else:
+        raise ValueError(
+            "'n_entries_in_table' can either be 'None' or an integer bigger than one."
+        )
+    return out
+
+
 @hookimpl
 def pytask_post_parse(config):
     live_manager = LiveManager()
     config["pm"].register(live_manager, "live_manager")
 
     if config["verbose"] >= 1:
-        live_execution = LiveExecution(live_manager, config["paths"], config["verbose"])
+        live_execution = LiveExecution(
+            live_manager,
+            config["paths"],
+            config["n_entries_in_table"],
+            config["verbose"],
+        )
         config["pm"].register(live_execution)
 
     live_collection = LiveCollection(live_manager)
@@ -66,6 +115,7 @@ class LiveExecution:
 
     _live_manager = attr.ib(type=LiveManager)
     _paths = attr.ib(type=Path)
+    _n_entries_in_table = attr.ib(type=int)
     _verbose = attr.ib(type=int)
     _running_tasks = attr.ib(factory=set)
     _reports = attr.ib(factory=list)
@@ -74,6 +124,7 @@ class LiveExecution:
     def pytask_execute_build(self):
         self._live_manager.start()
         yield
+        self._update_table(reduce_table=False)
         self._live_manager.stop(transient=False)
 
     @hookimpl(tryfirst=True)
@@ -86,10 +137,27 @@ def pytask_execute_task_log_end(self, report):
         self.update_reports(report)
         return True
 
-    def _generate_table(self):
+    def _generate_table(self, reduce_table: bool) -> Union[None, Table]:
+        """Generate the table.
+
+        First, display all completed tasks and, then, all running tasks.
+
+        The number of entries can be limited. All running tasks are always displayed and
+        if more entries are requested, the list is filled up with completed tasks.
+
+        """
         if self._running_tasks or self._reports:
+
+            n_reports_to_display = self._n_entries_in_table - len(self._running_tasks)
+            if not reduce_table:
+                relevant_reports = self._reports
+            elif n_reports_to_display >= 1:
+                relevant_reports = self._reports[-n_reports_to_display:]
+            else:
+                relevant_reports = []
+
             table = Table("Task", "Outcome")
-            for report in self._reports:
+            for report in relevant_reports:
                 if report["symbol"] in ("s", "p") and self._verbose < 2:
                     pass
                 else:
@@ -103,8 +171,8 @@ def _generate_table(self):
 
         return table
 
-    def _update_table(self):
-        table = self._generate_table()
+    def _update_table(self, reduce_table: bool = True):
+        table = self._generate_table(reduce_table)
         self._live_manager.update(table)
 
     def update_running_tasks(self, new_running_task):

Diff for: tests/test_live.py

@@ -1,13 +1,43 @@
 import textwrap
+from contextlib import ExitStack as does_not_raise  # noqa: N813
 
 import pytest
+from _pytask.live import _parse_n_entries_in_table
 from _pytask.live import LiveExecution
 from _pytask.live import LiveManager
 from _pytask.nodes import PythonFunctionTask
 from _pytask.report import ExecutionReport
 from pytask import cli
 
 
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    "value, expectation, expected",
+    [
+        pytest.param(None, does_not_raise(), None, id="none parsed"),
+        pytest.param(3, does_not_raise(), 3, id="int parsed"),
+        pytest.param("10", does_not_raise(), 10, id="string int parsed"),
+        pytest.param("all", does_not_raise(), 1_000_000, id="all to large int"),
+        pytest.param(
+            -1,
+            pytest.raises(ValueError, match="'n_entries_in_table' can"),
+            None,
+            id="negative int raises error",
+        ),
+        pytest.param(
+            "-1",
+            pytest.raises(ValueError, match="'n_entries_in_table' can"),
+            None,
+            id="negative int raises error",
+        ),
+    ],
+)
+def test_parse_n_entries_in_table(value, expectation, expected):
+    with expectation:
+        result = _parse_n_entries_in_table(value)
+        assert result == expected
+
+
 @pytest.mark.end_to_end
 @pytest.mark.parametrize("verbose", [False, True])
 def test_verbose_mode_execution(tmp_path, runner, verbose):
@@ -33,7 +63,7 @@ def test_live_execution_sequentially(capsys, tmp_path):
     )
 
     live_manager = LiveManager()
-    live = LiveExecution(live_manager, [tmp_path], 1)
+    live = LiveExecution(live_manager, [tmp_path], 20, 1)
 
     live_manager.start()
     live.update_running_tasks(task)
@@ -86,7 +116,7 @@ def test_live_execution_displays_skips_and_persists(capsys, tmp_path, verbose, s
     )
 
     live_manager = LiveManager()
-    live = LiveExecution(live_manager, [tmp_path], verbose)
+    live = LiveExecution(live_manager, [tmp_path], 20, verbose)
 
     live_manager.start()
     live.update_running_tasks(task)
@@ -113,3 +143,72 @@ def test_live_execution_displays_skips_and_persists(capsys, tmp_path, verbose, s
         assert f"│ {symbol}" in captured.out
 
     assert "running" not in captured.out
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize("n_entries_in_table", [1, 2])
+def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_table):
+    path = tmp_path.joinpath("task_dummy.py")
+    running_task = PythonFunctionTask(
+        "task_running", path.as_posix() + "::task_running", path, None
+    )
+
+    live_manager = LiveManager()
+    live = LiveExecution(live_manager, [tmp_path], n_entries_in_table, 1)
+
+    live_manager.start()
+    live.update_running_tasks(running_task)
+    live_manager.stop()
+
+    captured = capsys.readouterr()
+    assert "Task" in captured.out
+    assert "Outcome" in captured.out
+    assert "::task_running" in captured.out
+    assert " running " in captured.out
+
+    completed_task = PythonFunctionTask(
+        "task_completed", path.as_posix() + "::task_completed", path, None
+    )
+    live.update_running_tasks(completed_task)
+    report = ExecutionReport(
+        task=completed_task, success=True, exc_info=None, symbol=".", color="black"
+    )
+
+    live_manager.resume()
+    live.update_reports(report)
+    live_manager.stop()
+
+    # Test that report is or is not included.
+    captured = capsys.readouterr()
+    assert "Task" in captured.out
+    assert "Outcome" in captured.out
+    assert "::task_running" in captured.out
+    assert " running " in captured.out
+
+    if n_entries_in_table == 1:
+        assert "task_dummy.py::task_completed" not in captured.out
+        assert "│ ." not in captured.out
+    else:
+        assert "task_dummy.py::task_completed" in captured.out
+        assert "│ ." in captured.out
+
+
+def test_full_execution_table_is_displayed_at_the_end_of_execution(tmp_path, runner):
+    source = """
+    import pytask
+
+    @pytask.mark.parametrize("produces", [f"{i}.txt" for i in range(4)])
+    def task_create_file(produces):
+        produces.touch()
+    """
+    # Subfolder to reduce task id and be able to check the output later.
+    tmp_path.joinpath("d").mkdir()
+    tmp_path.joinpath("d", "task_t.py").write_text(textwrap.dedent(source))
+
+    result = runner.invoke(
+        cli, [tmp_path.joinpath("d").as_posix(), "--n-entries-in-table=1"]
+    )
+
+    assert result.exit_code == 0
+    for i in range(4):
+        assert f"{i}.txt" in result.output

Diff for: tests/test_mark.py

@@ -24,7 +24,7 @@ def test_pytask_mark_notcallable() -> None:
 @pytest.mark.unit
 @pytest.mark.filterwarnings("ignore:Unknown pytask.mark.foo")
 def test_mark_with_param():
-    def some_function(abc):
+    def some_function():
         pass
 
     class SomeClass:

Diff for: tests/test_mark_expression.py

@@ -11,10 +11,10 @@ def evaluate(input_: str, matcher: Callable[[str], bool]) -> bool:
 
 @pytest.mark.unit
 def test_empty_is_false() -> None:
-    assert not evaluate("", lambda ident: False)
-    assert not evaluate("", lambda ident: True)
-    assert not evaluate("   ", lambda ident: False)
-    assert not evaluate("\t", lambda ident: False)
+    assert not evaluate("", lambda ident: False)  # noqa: U100
+    assert not evaluate("", lambda ident: True)  # noqa: U100
+    assert not evaluate("   ", lambda ident: False)  # noqa: U100
+    assert not evaluate("\t", lambda ident: False)  # noqa: U100
 
 
 @pytest.mark.unit
@@ -119,7 +119,7 @@ def test_syntax_oddeties(expr: str, expected: bool) -> None:
 )
 def test_syntax_errors(expr: str, column: int, message: str) -> None:
     with pytest.raises(ParseError) as excinfo:
-        evaluate(expr, lambda ident: True)
+        evaluate(expr, lambda ident: True)  # noqa: U100
     assert excinfo.value.column == column
     assert excinfo.value.message == message
 
@@ -183,4 +183,4 @@ def test_valid_idents(ident: str) -> None:
 )
 def test_invalid_idents(ident: str) -> None:
     with pytest.raises(ParseError):
-        evaluate(ident, lambda ident: True)
+        evaluate(ident, lambda ident: True)  # noqa: U100

Diff for: tests/test_parametrize.py

@@ -33,7 +33,7 @@ def session():
 @pytest.mark.integration
 def test_pytask_generate_tasks_0(session):
     @pytask.mark.parametrize("i", range(2))
-    def func(i):
+    def func(i):  # noqa: U100
         pass
 
     names_and_objs = pytask_parametrize_task(session, "func", func)
@@ -48,7 +48,7 @@ def func(i):
 def test_pytask_generate_tasks_1(session):
     @pytask.mark.parametrize("j", range(2))
     @pytask.mark.parametrize("i", range(2))
-    def func(i, j):
+    def func(i, j):  # noqa: U100
         pass
 
     names_and_objs = pytask_parametrize_task(session, "func", func)
@@ -66,7 +66,7 @@ def func(i, j):
 def test_pytask_generate_tasks_2(session):
     @pytask.mark.parametrize("j, k", itertools.product(range(2), range(2)))
     @pytask.mark.parametrize("i", range(2))
-    def func(i, j, k):
+    def func(i, j, k):  # noqa: U100
         pass
 
     names_and_objs = pytask_parametrize_task(session, "func", func)