feat(pytest): fallback to summary info

rcarriga · rcarriga · commit efd96a033c2b · 2021-11-28T17:13:16.000Z
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,3 @@
-mypy
 pytest
 pytest-cov
 pytest-asyncio
diff --git a/rplugin/python3/ultest/handler/parsers/output/python/pytest.py b/rplugin/python3/ultest/handler/parsers/output/python/pytest.py
@@ -1,11 +1,14 @@
 from dataclasses import dataclass
+from logging import getLogger
 from typing import List, Optional
 
 from .. import parsec as p
 from ..base import ParsedOutput, ParseResult
 from ..parsec import generate
 from ..util import eol, join_chars, until_eol
 
+logger = getLogger(__name__)
+
 
 @dataclass
 class PytestCodeTrace:
@@ -18,15 +21,41 @@ class PytestCodeTrace:
 @generate
 def pytest_output():
     yield pytest_test_results_summary
-    failed = yield p.many(failed_test_section)
-    yield pytest_summary_info
-    return ParsedOutput(results=failed)
+    parsed_outputs = yield p.many1(failed_test_section)
+    parsed_summary = yield pytest_summary_info
+    parsed_results = {
+        (r.file, r.name, *r.namespaces): r
+        for r in [*parsed_summary, *parsed_outputs]
+        if r
+    }
+    return ParsedOutput(results=list(parsed_results.values()))
 
 
 @generate
 def failed_test_section():
     namespaces, test_name = yield failed_test_section_title
     yield until_eol
+    raw_output_lines = yield p.many1(
+        p.exclude(until_eol, failed_test_section_title ^ pytest_summary_info_title)
+    )
+    output_text = "\n".join(raw_output_lines) + "\n"
+    try:
+
+        file, err_msg, err_line = failed_test_section_output.parse(output_text)
+        return ParseResult(
+            name=test_name,
+            namespaces=namespaces,
+            file=file,
+            message=err_msg,
+            line=err_line,
+        )
+    except Exception as e:
+        logger.debug(f"Failed to parse output: {e}\n----\n{output_text}\n----")
+        return None
+
+
+@generate
+def failed_test_section_output():
     traces: List[PytestCodeTrace]
     traces = yield failed_test_code_sections
     sections = yield failed_test_captured_output_sections
@@ -40,26 +69,24 @@ def failed_test_section():
     for trace in traces:
         if trace.message:
             error_message = trace.message
-    return ParseResult(
-        name=test_name,
-        namespaces=namespaces,
-        file=test_file,
-        message=error_message,
-        line=test_line_no,
+    return (
+        test_file,
+        error_message,
+        test_line_no,
     )
 
 
 @generate
 def failed_test_section_title():
-    yield p.many1(p.string("_")) >> p.space()
+    yield p.string("_") >> p.many1(p.string("_")) >> p.space()
     name_elements = (
         yield p.many1(p.none_of(" "))
         .parsecmap(join_chars)
         .parsecmap(lambda elems: elems.split("."))
     )
     namespaces = name_elements[:-1]
     test_name = name_elements[-1]
-    yield p.space() >> p.many1(p.string("_"))
+    yield until_eol
     return (namespaces, test_name)
 
 
@@ -74,8 +101,7 @@ def failed_test_captured_output_sections():
 
 @generate
 def failed_test_code_sections():
-
-    sections = yield p.sepBy(failed_test_code_section, failed_test_section_sep)
+    sections = yield p.sepBy1(failed_test_code_section, failed_test_section_sep)
     return sections
 
 
@@ -125,11 +151,27 @@ def failed_test_error_message_line():
     return error_text
 
 
+@generate
+def pytest_summary_failed_test():
+    yield p.string("FAILED ")
+    names = yield p.sepBy1(
+        p.many1(p.none_of(": ")).parsecmap(join_chars), p.string("::")
+    )
+    file, *namespaces = names
+    yield until_eol
+    return ParseResult(
+        name=namespaces[-1],
+        namespaces=namespaces[:-1],
+        file=file,
+    )
+
+
 @generate
 def pytest_summary_info():
     yield pytest_summary_info_title
-    summary = yield p.many(until_eol)
-    return summary
+    parsed_summary = yield p.many1(pytest_summary_failed_test)
+    yield p.many(until_eol)
+    return parsed_summary
 
 
 @generate
diff --git a/tests/mocks/test_outputs/pytest_hypothesis_2 b/tests/mocks/test_outputs/pytest_hypothesis_2
@@ -0,0 +1,65 @@
+[1m================================================= test session starts ==================================================[0m
+platform linux -- Python 3.8.6, pytest-6.2.5, py-1.10.0, pluggy-1.0.0
+rootdir: /home/ronan/Dev/repos/hypothesis, configfile: pytest.ini
+plugins: xdist-2.4.0, forked-1.3.0, hypothesis-6.24.0
+[1mcollecting ... [0m[1mcollected 1 item                                                                                                       [0m
+
+tests/numpy/test_from_dtype.py [31mF[0m[31m                                                                                 [100%][0m
+
+======================================================= FAILURES =======================================================
+[31m[1m______________________________________________ test_can_cast_for_scalars _______________________________________________[0m
+Traceback (most recent call last):
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/tests/numpy/test_from_dtype.py", line 80, in test_can_cast_for_scalars
+  def test_can_cast_for_scalars(data):
+    File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/core.py", line 1199, in wrapped_test
+    raise the_error_hypothesis_found
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/core.py", line 1168, in wrapped_test
+    state.run_engine()
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/core.py", line 780, in run_engine
+    runner.run()
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/internal/conjecture/engine.py", line 475, in run
+    self._run()
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/internal/conjecture/engine.py", line 877, in _run
+    self.generate_new_examples()
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/internal/conjecture/engine.py", line 609, in generate_new_examples
+    zero_data = self.cached_test_function(bytes(BUFFER_SIZE))
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/internal/conjecture/engine.py", line 1056, in cached_test_function
+    self.test_function(data)
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/internal/conjecture/engine.py", line 213, in test_function
+    self.__stoppable_test_function(data)
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/internal/conjecture/engine.py", line 189, in __stoppable_test_function
+    self._test_function(data)
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/core.py", line 727, in _execute_once_for_engine
+    escalate_hypothesis_internal_error()
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/core.py", line 701, in _execute_once_for_engine
+    result = self.execute_once(data)
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/core.py", line 639, in execute_once
+    result = self.test_runner(data, run)
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/executors.py", line 52, in default_new_style_executor
+    return function(data)
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/core.py", line 635, in run
+    return test(*args, **kwargs)
+    File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/tests/numpy/test_from_dtype.py", line 80, in test_can_cast_for_scalars
+    def test_can_cast_for_scalars(data):
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/core.py", line 577, in test
+  result = self.test(*args, **kwargs)
+  File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/tests/numpy/test_from_dtype.py", line 87, in test_can_cast_for_scalars
+  result = data.draw(
+      File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/strategies/_internal/core.py", line 1692, in draw
+      result = self.conjecture_data.draw(strategy)
+      File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/internal/conjecture/data.py", line 866, in draw
+      strategy.validate()
+      File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py", line 401, in validate
+      self.do_validate()
+      File "/home/ronan/Dev/repos/hypothesis/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py", line 134, in do_validate
+      assert isinstance(w, SearchStrategy), f"{self!r} returned non-strategy {w!r}"
+      AssertionError: arrays(dtype=dtype('int16'), shape=(), elements=from_dtype(dtype('bool'))) returned non-strategy []
+      ------------------------------------------------------ Hypothesis ------------------------------------------------------
+      You can add @seed(11024453522097809882419571698055639517) to this test or run pytest with --hypothesis-seed=11024453522097809882419571698055639517 to reproduce this failure.
+      ================================================= slowest 20 durations =================================================
+      0.01s setup    hypothesis-python/tests/numpy/test_from_dtype.py::test_can_cast_for_scalars
+
+      (2 durations < 0.005s hidden.  Use -vv to show these durations.)
+      =============================================== short test summary info ================================================
+      FAILED tests/numpy/test_from_dtype.py::test_can_cast_for_scalars - AssertionError: arrays(dtype=dtype('int16'), shape...
+        [31m================================================== [31m[1m1 failed[0m[31m in 0.06s[0m[31m ===================================================[0m
diff --git a/tests/unit/handler/parsers/output/python/test_pytest.py b/tests/unit/handler/parsers/output/python/test_pytest.py
@@ -6,6 +6,7 @@
     failed_test_section,
     failed_test_section_error_message,
     failed_test_section_title,
+    pytest_summary_info,
 )
 from tests.mocks import get_output
 
@@ -105,13 +106,41 @@ def test_parse_hypothesis_file(self):
             ],
         )
 
+    def test_parse_summary_if_output_not_parsable(self):
+        raw = """
+================================================= test session starts ==================================================
+...
+
+======================================================= FAILURES =======================================================
+______________________________________________ test_can_cast_for_scalars _______________________________________________
+This is not parsable
+------------------------------------------------------ Hypothesis ------------------------------------------------------
+You can add @seed(11024453522097809882419571698055639517) to this test or run pytest with --hypothesis-seed=11024453522097809882419571698055639517 to reproduce this failure.
+================================================= slowest 20 durations =================================================
+0.01s setup    hypothesis-python/tests/numpy/test_from_dtype.py::test_can_cast_for_scalars
+
+(2 durations < 0.005s hidden.  Use -vv to show these durations.)
+=============================================== short test summary info ================================================
+FAILED tests/numpy/test_from_dtype.py::test_can_cast_for_scalars - AssertionError: arrays(dtype=dtype('int16'), shape...
+================================================== 1 failed in 0.06s ===================================================
+"""
+        expected = [
+            ParseResult(
+                name="test_can_cast_for_scalars",
+                namespaces=[],
+                file="tests/numpy/test_from_dtype.py",
+            )
+        ]
+        parser = OutputParser([])
+        result = parser.parse_failed("python#pytest", raw)
+        self.assertEqual(result, expected)
+
     def test_parse_failed_test_section_title(self):
-        raw = "_____ MyClass.test_a ______"
+        raw = "_____ MyClass.test_a ______\n"
         result = failed_test_section_title.parse(raw)
         self.assertEqual(result, (["MyClass"], "test_a"))
 
     def test_parse_failed_test_section_error(self):
-        self.maxDiff = None
         raw = """E       AssertionError: {'a': 1, 'b': 2, 'c': 3} != {'a': 1, 'b': 5, 'c': 3, 'd': 4}
 E       - {'a': 1, 'b': 2, 'c': 3}
 E       ?               ^
@@ -130,6 +159,69 @@ def test_parse_failed_test_section_error(self):
         ]
         self.assertEqual(expected, result)
 
+    def test_parse_failed_test_short_summary(self):
+        raw = """======================================================================================================================== short test summary info =========================================================================================================================
+FAILED test_a.py::test_b - Exception: OH NO
+FAILED test_a.py::TestClass::test_b - AssertionError: {'a': 1, 'b': 2, 'c': 3} != {'a': 1, 'b': 5, 'c': 3, 'd': 4}
+FAILED test_b.py::AnotherClass::test_a - AssertionError: assert 2 == 3
+FAILED test_b.py::test_d - assert 2 == 3
+FAILED subtests/test_c.py::TestStuff::test_thing_2 - AssertionError: assert False
+FAILED subtests/test_c.py::test_a - assert False
+====================================================================================================================== 6 failed, 5 passed in 0.39s =======================================================================================================================
+"""
+        result = pytest_summary_info.parse(raw)
+        expected = [
+            ParseResult(
+                name="test_b",
+                namespaces=[],
+                file="test_a.py",
+                message=None,
+                output=None,
+                line=None,
+            ),
+            ParseResult(
+                name="test_b",
+                namespaces=["TestClass"],
+                file="test_a.py",
+                message=None,
+                output=None,
+                line=None,
+            ),
+            ParseResult(
+                name="test_a",
+                namespaces=["AnotherClass"],
+                file="test_b.py",
+                message=None,
+                output=None,
+                line=None,
+            ),
+            ParseResult(
+                name="test_d",
+                namespaces=[],
+                file="test_b.py",
+                message=None,
+                output=None,
+                line=None,
+            ),
+            ParseResult(
+                name="test_thing_2",
+                namespaces=["TestStuff"],
+                file="subtests/test_c.py",
+                message=None,
+                output=None,
+                line=None,
+            ),
+            ParseResult(
+                name="test_a",
+                namespaces=[],
+                file="subtests/test_c.py",
+                message=None,
+                output=None,
+                line=None,
+            ),
+        ]
+        self.assertEqual(expected, result)
+
     def test_parse_failed_test_section(self):
         raw = """_____________________________________________________________________ MyClass.test_b _____________________________________________________________________
 
diff --git a/tests/unit/models/test_tree.py b/tests/unit/models/test_tree.py
@@ -1,4 +1,3 @@
-import logging
 import random
 from typing import List, Union
 

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-mypy`
`2`	`1`	`pytest`
`3`	`2`	`pytest-cov`
`4`	`3`	`pytest-asyncio`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-import logging`
`2`	`1`	`import random`
`3`	`2`	`from typing import List, Union`
`4`	`3`