Update report handler to report list summaries of test results

haas/plugins/result_handler.py

@@ -63,14 +63,38 @@ def sort(items):
     return list(sort(handlers))
 
 
+def _summary_report_config(args=None):
+    """ Get the summary report config based on arguments
+    """
+    report_lists = [TestCompletionStatus.error, TestCompletionStatus.failure]
+    if args is None or args.list_summary is None:
+        return report_lists
+    if 'skipped' in args.list_summary:
+        report_lists.append(TestCompletionStatus.skipped)
+    return report_lists
+
+
 class QuietTestResultHandler(IResultHandlerPlugin):
     separator1 = '=' * 70
     separator2 = separator2
+    OPTION_DEFAULT = object()
 
-    def __init__(self, test_count):
+    _summary_formats = {
+        TestCompletionStatus.failure: 'FAIL',
+        TestCompletionStatus.error: 'ERROR',
+        TestCompletionStatus.unexpected_success: 'unexpected success',
+        TestCompletionStatus.expected_failure: 'expected failure',
+        TestCompletionStatus.skipped: 'skipped',
+    }
+
+    def __init__(self, test_count, report_lists=None):
         self.enabled = True
         self.stream = _WritelnDecorator(sys.stderr)
         self._test_count = test_count
+        if report_lists is None:
+            self._report_lists = _summary_report_config()
+        else:
+            self._report_lists = report_lists
         self.tests_run = 0
         self.descriptions = True
         self.expectedFailures = expectedFailures = []
@@ -90,12 +114,15 @@ def __init__(self, test_count):
 
     @classmethod
     def from_args(cls, args, name, dest_prefix, test_count):
+        report_lists = _summary_report_config(args)
         if args.verbosity == 0:
-            return cls(test_count=test_count)
+            return cls(test_count=test_count, report_lists=report_lists)
 
     @classmethod
-    def add_parser_arguments(self, parser, name, option_prefix, dest_prefix):
-        pass
+    def add_parser_arguments(cls, parser, name, option_prefix, dest_prefix):
+        parser.add_argument(
+            '--list-summary', nargs='?',
+            help=('Report test list summary'))
 
     def get_test_description(self, test):
         return get_test_description(test, descriptions=self.descriptions)
@@ -111,38 +138,12 @@ def start_test_run(self):
 
     def stop_test_run(self):
         self.stop_time = time.time()
-        self.print_errors()
+        for status in self._report_lists:
+            self.print_list_summary(status)
         self.print_summary()
 
-    def print_errors(self):
-        """Print all errors and failures to the console.
-
-        """
-        self.stream.writeln()
-        self.print_error_list('ERROR', self.errors)
-        self.print_error_list('FAIL', self.failures)
-
-    def print_error_list(self, error_kind, errors):
-        """Print the list of errors or failures.
-
-        Parameters
-        ----------
-        error_kind : str
-            ``'ERROR'`` or ``'FAIL'``
-        errors : list
-            List of :class:`~haas.result.TestResult`
-
-        """
-        for result in errors:
-            self.stream.writeln(self.separator1)
-            self.stream.writeln(
-                '%s: %s' % (error_kind, self.get_test_description(
-                    result.test)))
-            self.stream.writeln(self.separator2)
-            self.stream.writeln(result.exception)
-
     def print_summary(self):
-        self.stream.writeln(self.separator2)
+        self.stream.writeln('\n' + self.separator2)
         time_taken = self.stop_time - self.start_time
 
         run = self.tests_run
@@ -173,10 +174,43 @@ def print_summary(self):
         if unexpectedSuccesses:
             infos.append("unexpected successes=%d" % unexpectedSuccesses)
         if infos:
-            self.stream.writeln(" (%s)" % (", ".join(infos),))
+            self.stream.writeln(" (%s)\n" % (", ".join(infos),))
         else:
             self.stream.write("\n")
 
+    def print_list_summary(self, status):
+        """Print the list of tests of a given status.
+
+        Parameters
+        ----------
+        status : TestCompletionStatus
+
+        """
+        results = self._collectors[status]
+        if len(results) == 0:
+            return
+
+        stream = self.stream
+        stream.writeln("\n")
+        template = self._summary_formats[status] + ': %s'
+        raised_exception = status in (
+            TestCompletionStatus.error, TestCompletionStatus.failure)
+        if not raised_exception:
+            stream.writeln(self.separator1)
+        for result in results:
+            if raised_exception:
+                stream.writeln(self.separator1)
+                stream.writeln(
+                    template % self.get_test_description(result.test))
+                stream.writeln(self.separator2)
+                stream.writeln(result.exception)
+            else:
+                stream.writeln(
+                    template % self.get_test_description(result.test))
+        if not raised_exception:
+            stream.writeln(self.separator2)
+
+
     def was_successful(self):
         return (len(self.errors) == 0 and
                 len(self.failures) == 0 and
@@ -201,8 +235,9 @@ class StandardTestResultHandler(QuietTestResultHandler):
 
     @classmethod
     def from_args(cls, args, name, dest_prefix, test_count):
+        report_lists = _summary_report_config(args)
         if args.verbosity == 1:
-            return cls(test_count=test_count)
+            return cls(test_count=test_count, report_lists=report_lists)
 
     def __call__(self, result):
         super(StandardTestResultHandler, self).__call__(result)
@@ -223,8 +258,9 @@ class VerboseTestResultHandler(StandardTestResultHandler):
 
     @classmethod
     def from_args(cls, args, name, dest_prefix, test_count):
+        report_lists = _summary_report_config(args)
         if args.verbosity == 2:
-            return cls(test_count=test_count)
+            return cls(test_count=test_count, report_lists=report_lists)
 
     def start_test(self, test):
         super(VerboseTestResultHandler, self).start_test(test)

haas/tests/test_buffering.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2013-2019 Simon Jagoe
+# All rights reserved.
+#
+# This software may be modified and distributed under the terms
+# of the 3-clause BSD license.  See the LICENSE.txt file for details.
+from __future__ import absolute_import, unicode_literals
+
+from datetime import datetime, timedelta
+import sys
+
+from mock import Mock, patch
+from six.moves import StringIO
+
+from ..plugins.i_result_handler_plugin import IResultHandlerPlugin
+from ..result import (
+    ResultCollector, TestResult, TestCompletionStatus, TestDuration
+)
+from ..testing import unittest
+from . import _test_cases
+from .fixtures import ExcInfoFixture, MockDateTime
+
+
+class TestBuffering(ExcInfoFixture, unittest.TestCase):
+
+    @patch('sys.stderr', new_callable=StringIO)
+    def test_buffering_stderr(self, stderr):
+        # Given
+        handler = Mock(spec=IResultHandlerPlugin)
+        collector = ResultCollector(buffer=True)
+        collector.add_result_handler(handler)
+        test_stderr = 'My Test Output'
+        start_time = datetime(2015, 12, 23, 8, 14, 12)
+        duration = timedelta(seconds=10)
+        end_time = start_time + duration
+        expected_duration = TestDuration(start_time, end_time)
+        case = _test_cases.TestCase('test_method')
+
+        # When
+        with patch('haas.result.datetime', new=MockDateTime(start_time)):
+            collector.startTest(case)
+
+        # Then
+        self.assertTrue(handler.start_test.called)
+        handler.start_test.reset_mock()
+
+        # When
+        sys.stderr.write(test_stderr)
+
+        # Then
+        self.assertEqual(stderr.getvalue(), '')
+
+        # Given
+        with self.exc_info(RuntimeError) as exc_info:
+            expected_result = TestResult.from_test_case(
+                case, TestCompletionStatus.error, expected_duration,
+                exception=exc_info, stderr=test_stderr)
+            # When
+            with patch('haas.result.datetime', new=MockDateTime(end_time)):
+                collector.addError(case, exc_info)
+        collector.stopTest(case)
+
+        # Then
+        self.assertIn(test_stderr, expected_result.exception)
+        handler.assert_called_once_with(expected_result)
+
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_buffering_stdout(self, stdout):
+        # Given
+        handler = Mock(spec=IResultHandlerPlugin)
+        collector = ResultCollector(buffer=True)
+        collector.add_result_handler(handler)
+        test_stdout = 'My Test Output'
+        start_time = datetime(2015, 12, 23, 8, 14, 12)
+        duration = timedelta(seconds=10)
+        end_time = start_time + duration
+        expected_duration = TestDuration(start_time, end_time)
+        case = _test_cases.TestCase('test_method')
+
+        # When
+        with patch('haas.result.datetime', new=MockDateTime(start_time)):
+            collector.startTest(case)
+
+        # Then
+        self.assertTrue(handler.start_test.called)
+        handler.start_test.reset_mock()
+
+        # When
+        sys.stdout.write(test_stdout)
+
+        # Then
+        self.assertEqual(stdout.getvalue(), '')
+
+        # Given
+        with self.exc_info(RuntimeError) as exc_info:
+            expected_result = TestResult.from_test_case(
+                case, TestCompletionStatus.error, expected_duration,
+                exception=exc_info, stdout=test_stdout)
+
+            # When
+            with patch('haas.result.datetime', new=MockDateTime(end_time)):
+                collector.addError(case, exc_info)
+        collector.stopTest(case)
+
+        # Then
+        self.assertIn(test_stdout, expected_result.exception)
+        handler.assert_called_once_with(expected_result)

haas/tests/test_deprecations.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2013-2019 Simon Jagoe
+# All rights reserved.
+#
+# This software may be modified and distributed under the terms
+# of the 3-clause BSD license.  See the LICENSE.txt file for details.
+from __future__ import absolute_import, unicode_literals
+
+from testfixtures import ShouldWarn
+
+from ..result import ResultCollecter
+from ..testing import unittest
+
+
+class TestResultCollecterDepricated(unittest.TestCase):
+
+    def test_deprecation_warning(self):
+        # Given
+        expected_warning = DeprecationWarning(
+            'ResultCollecter is deprecated in favour of ResultCollector and '
+            'will be removed in the next release.',
+        )
+
+        # When/Then
+        with ShouldWarn(expected_warning):
+            ResultCollecter()