feat(output): structured unittest support

Author: rcarriga

lua/ultest/diagnostic/init.lua

@@ -79,7 +79,7 @@ local function draw_buffer(file)
   local results = api.nvim_buf_get_var(bufnr, "ultest_results")
 
   local valid_results = vim.tbl_filter(function(result)
-    return result.error_line and result.error_message
+    return type(result) == "table" and result.error_line and result.error_message
   end, results)
 
   local diagnostics = create_diagnostics(bufnr, valid_results)

rplugin/python3/ultest/handler/parsers/output/__init__.py

@@ -6,6 +6,7 @@
 from .base import ParseResult
 from .parsec import ParseError
 from .python.pytest import pytest_output
+from .python.unittest import unittest_output
 
 
 @dataclass
@@ -17,10 +18,6 @@ class OutputPatterns:
 
 
 _BASE_PATTERNS = {
-    "python#pyunit": OutputPatterns(
-        failed_test=r"^FAIL: (?P<name>.*) \(.*?(?P<namespaces>\..+)\)",
-        namespace_separator=r"\.",
-    ),
     "go#gotest": OutputPatterns(failed_test=r"^.*--- FAIL: (?P<name>.+?) "),
     "go#richgo": OutputPatterns(
         failed_test=r"^FAIL\s\|\s(?P<name>.+?) \(.*\)",
@@ -43,7 +40,10 @@ class OutputPatterns:
 
 class OutputParser:
     def __init__(self, disable_patterns: List[str]) -> None:
-        self._parsers = {"python#pytest": pytest_output}
+        self._parsers = {
+            "python#pytest": pytest_output,
+            "python#pyunit": unittest_output,
+        }
         self._patterns = {
             runner: patterns
             for runner, patterns in _BASE_PATTERNS.items()

rplugin/python3/ultest/handler/parsers/output/python/pytest.py

@@ -1,8 +1,7 @@
 from .. import parsec as p
 from ..base import ParsedOutput, ParseResult
 from ..parsec import generate
-
-join_chars = lambda chars: "".join(chars)
+from ..util import join_chars, until_eol
 
 
 @generate
@@ -107,19 +106,6 @@ def pytest_test_results_summary():
     return summary
 
 
-@generate
-def eol():
-    new_line = yield p.string("\r\n") ^ p.string("\n")
-    return new_line
-
-
-@generate
-def until_eol():
-    text = yield p.many(p.exclude(p.any(), eol)).parsecmap(join_chars)
-    yield eol
-    return text
-
-
 @generate
 def failed_test_error_location():
     file_name = yield p.many1(p.none_of(" :")).parsecmap(join_chars)

rplugin/python3/ultest/handler/parsers/output/python/unittest.py

@@ -0,0 +1,90 @@
+from .. import parsec as p
+from ..base import ParsedOutput, ParseResult
+from ..parsec import generate
+from ..util import eol, join_chars, until_eol
+
+
+class ErroredTestError(Exception):
+    ...
+
+
+@generate
+def unittest_output():
+    try:
+        yield p.many(p.exclude(p.any(), failed_test_title))
+        failed_tests = yield p.many1(failed_test)
+        yield p.many(p.any())
+        return ParsedOutput(results=failed_tests)
+    except ErroredTestError:
+        return ParsedOutput(results=[])
+
+
+@generate
+def failed_test():
+    name, namespace = yield failed_test_title
+    file, error_line = yield failed_test_traceback
+    error_message = yield failed_test_error_message
+    return ParseResult(
+        name=name,
+        namespaces=[namespace],
+        file=file,
+        message=error_message,
+        line=error_line,
+    )
+
+
+@generate
+def failed_test_title():
+    text = (
+        yield p.many1(p.string("="))
+        >> eol
+        >> (p.string("FAIL") ^ p.string("ERROR"))
+        << p.string(": ")
+    )
+    test = yield p.many1(p.none_of(" ")).parsecmap(join_chars)
+    yield p.space()
+    namespace = (
+        yield (p.string("(") >> p.many1(p.none_of(")")) << (p.string(")") >> until_eol))
+        .parsecmap(join_chars)
+        .parsecmap(lambda s: s.split(".")[-1])
+    )
+    if namespace == "_FailedTest":
+        # Can't infer namespace from file that couldn't be imported
+        raise ErroredTestError
+    yield p.many1(p.string("-")) >> eol
+    return test, namespace
+
+
+@generate
+def traceback_location():
+    file = (
+        yield p.spaces()
+        >> p.string('File "')
+        >> p.many1(p.none_of('"')).parsecmap(join_chars)
+        << p.string('"')
+    )
+    line = yield (
+        p.string(", line ")
+        >> p.many1(p.digit()).parsecmap(join_chars).parsecmap(int)
+        << until_eol
+    )
+    return file, line
+
+
+@generate
+def failed_test_traceback():
+    yield p.string("Traceback") >> until_eol
+    file, line = yield traceback_location
+    yield p.many1(p.string(" ") >> until_eol)
+    return file, line
+
+
+@generate
+def failed_test_error_message():
+    message = yield p.many1(p.exclude(until_eol, p.string("--") ^ p.string("==")))
+    remove_index = len(message) - 0
+    for line in reversed(message):
+        if line != "":
+            break
+        remove_index -= 1
+    return message[:remove_index]  # Ends with blank lines

rplugin/python3/ultest/handler/parsers/output/util.py

@@ -0,0 +1,17 @@
+from . import parsec as p
+from .parsec import generate
+
+join_chars = lambda chars: "".join(chars)
+
+
+@generate
+def until_eol():
+    text = yield p.many(p.exclude(p.any(), eol)).parsecmap(join_chars)
+    yield eol
+    return text
+
+
+@generate
+def eol():
+    new_line = yield p.string("\r\n") ^ p.string("\n")
+    return new_line

scripts/style

@@ -8,7 +8,7 @@ if [[ $1 == "-w" ]]; then
   black "${PYTHON_DIRS[@]}"
   isort "${PYTHON_DIRS[@]}"
   autoflake --remove-unused-variables --remove-all-unused-imports --ignore-init-module-imports --remove-duplicate-keys --recursive -i "${PYTHON_DIRS[@]}"
-  find -name \*.lua -print0 | xargs -0 luafmt -w replace -i 2
+  stylua .
 else
   black --check "${PYTHON_DIRS[@]}"
   isort --check "${PYTHON_DIRS[@]}"

tests/mocks/test_outputs/pyunit

@@ -1,13 +1,46 @@
-F.
+F3
+EF.F
 ======================================================================
-FAIL: test_d (test_a.TestMyClass)
+ERROR: test_c (test_a.TestClass)
 ----------------------------------------------------------------------
 Traceback (most recent call last):
-  File "/home/ronan/tests/test_a.py", line 9, in test_d
-    assert 33 == 3
+  File "/home/ronan/tests/test_a.py", line 37, in test_c
+    a_function()
+  File "/home/ronan/tests/tests/__init__.py", line 6, in a_function
+    raise Exception
+Exception
+
+======================================================================
+FAIL: test_b (test_a.TestClass)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/home/ronan/tests/test_a.py", line 34, in test_b
+    self.assertEqual({"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 5, "c": 3, "d": 4})
+AssertionError: {'a': 1, 'b': 2, 'c': 3} != {'a': 1, 'b': 5, 'c': 3, 'd': 4}
+- {'a': 1, 'b': 2, 'c': 3}
+?               ^
+
++ {'a': 1, 'b': 5, 'c': 3, 'd': 4}
+?               ^        ++++++++
+
+
+======================================================================
+FAIL: test_a (test_b.AnotherClass)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/home/ronan/tests/test_b.py", line 7, in test_a
+    assert 2 == 3
+AssertionError
+
+======================================================================
+FAIL: test_thing (tests.test_c.TestStuff)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/home/ronan/tests/tests/test_c.py", line 6, in test_thing
+    assert False
 AssertionError
 
 ----------------------------------------------------------------------
-Ran 2 tests in 0.001s
+Ran 5 tests in 0.001s
 
-FAILED (failures=1)
+FAILED (failures=3, errors=1)

tests/unit/handler/parsers/output/python/test_unittest.py

@@ -0,0 +1,111 @@
+from unittest import TestCase
+
+from rplugin.python3.ultest.handler.parsers.output import OutputParser
+from rplugin.python3.ultest.handler.parsers.output.python.unittest import (
+    ErroredTestError,
+    ParseResult,
+    failed_test,
+)
+from tests.mocks import get_output
+
+
+class TestUnittestParser(TestCase):
+    def test_parse_failed_test(self):
+        raw = """======================================================================
+FAIL: test_b (test_a.TestClass)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/home/ronan/tests/test_a.py", line 34, in test_b
+    self.assertEqual({"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 5, "c": 3, "d": 4})
+AssertionError: {'a': 1, 'b': 2, 'c': 3} != {'a': 1, 'b': 5, 'c': 3, 'd': 4}
+- {'a': 1, 'b': 2, 'c': 3}
+?               ^
+
++ {'a': 1, 'b': 5, 'c': 3, 'd': 4}
+?               ^        ++++++++
+
+"""
+
+        expected = ParseResult(
+            name="test_b",
+            namespaces=["TestClass"],
+            file="/home/ronan/tests/test_a.py",
+            line=34,
+            message=[
+                "AssertionError: {'a': 1, 'b': 2, 'c': 3} != {'a': 1, 'b': 5, 'c': 3, 'd': 4}",
+                "- {'a': 1, 'b': 2, 'c': 3}",
+                "?               ^",
+                "",
+                "+ {'a': 1, 'b': 5, 'c': 3, 'd': 4}",
+                "?               ^        ++++++++",
+            ],
+        )
+        result = failed_test.parse(raw)
+        self.assertEqual(result, expected)
+
+    def test_parse_errored_test_raises(self):
+        raw = """======================================================================
+ERROR: test_c (unittest.loader._FailedTest)
+----------------------------------------------------------------------
+ImportError: Failed to import test module: test_c
+Traceback (most recent call last):
+  File "/home/ronan/.pyenv/versions/3.8.6/lib/python3.8/unittest/loader.py", line 436, in _find_test_path
+    module = self._get_module_from_name(name)
+  File "/home/ronan/.pyenv/versions/3.8.6/lib/python3.8/unittest/loader.py", line 377, in _get_module_from_name
+    __import__(name)
+  File "/home/ronan/tests/test_c.py", line 6, in <module>
+    class CTests(TestCase):
+  File "/home/ronan/tests/test_c.py", line 8, in CTests
+    @not_a_decorator
+NameError: name 'not_a_decorator' is not defined
+
+"""
+        with self.assertRaises(ErroredTestError):
+            failed_test.parse(raw)
+
+    def test_parse_unittest(self):
+        parser = OutputParser([])
+        raw = get_output("pyunit")
+        result = parser.parse_failed("python#pyunit", raw)
+        expected = [
+            ParseResult(
+                name="test_c",
+                namespaces=["TestClass"],
+                file="/home/ronan/tests/test_a.py",
+                message=["Exception"],
+                output=None,
+                line=37,
+            ),
+            ParseResult(
+                name="test_b",
+                namespaces=["TestClass"],
+                file="/home/ronan/tests/test_a.py",
+                message=[
+                    "AssertionError: {'a': 1, 'b': 2, 'c': 3} != {'a': 1, 'b': 5, 'c': 3, 'd': 4}",
+                    "- {'a': 1, 'b': 2, 'c': 3}",
+                    "?               ^",
+                    "",
+                    "+ {'a': 1, 'b': 5, 'c': 3, 'd': 4}",
+                    "?               ^        ++++++++",
+                ],
+                output=None,
+                line=34,
+            ),
+            ParseResult(
+                name="test_a",
+                namespaces=["AnotherClass"],
+                file="/home/ronan/tests/test_b.py",
+                message=["AssertionError"],
+                output=None,
+                line=7,
+            ),
+            ParseResult(
+                name="test_thing",
+                namespaces=["TestStuff"],
+                file="/home/ronan/tests/tests/test_c.py",
+                message=["AssertionError"],
+                output=None,
+                line=6,
+            ),
+        ]
+        self.assertEqual(expected, result)

tests/unit/handler/parsers/test_output.py

@@ -8,13 +8,6 @@ class TestOutputParser(TestCase):
     def setUp(self) -> None:
         self.parser = OutputParser([])
 
-    def test_parse_pyunit(self):
-        output = get_output("pyunit")
-        failed = list(self.parser.parse_failed("python#pyunit", output))
-        self.assertEqual(
-            failed, [ParseResult(file="", name="test_d", namespaces=["TestMyClass"])]
-        )
-
     def test_parse_gotest(self):
         output = get_output("gotest")
         failed = list(self.parser.parse_failed("go#gotest", output))