try_flag_unittest.py

# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import unittest

from blinkpy.common.host_mock import MockHost
from blinkpy.common.net.git_cl import BuildStatus
from blinkpy.common.net.git_cl_mock import MockGitCL
from blinkpy.common.net.results_fetcher import Build
from blinkpy.common.net.web_test_results import WebTestResults
from blinkpy.common.path_finder import PathFinder
from blinkpy.web_tests.try_flag import TryFlag


class TryFlagTest(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        self.linux_build = Build('linux-rel', 100)
        self.mac_build = Build('mac-rel', 101)
        self.win_build = Build('win7-rel', 102)
        self.mock_try_results = {
            self.linux_build: BuildStatus.SUCCESS,
            self.win_build: BuildStatus.SUCCESS,
            self.mac_build: BuildStatus.SUCCESS
        }
        super(TryFlagTest, self).__init__(*args, **kwargs)

    def _run_trigger_test(self, regenerate):
        host = MockHost()
        git = host.git()
        git_cl = MockGitCL(host)
        finder = PathFinder(host.filesystem)

        flag_file = finder.path_from_web_tests(
            'additional-driver-flag.setting')
        flag_expectations_file = finder.path_from_web_tests(
            'FlagExpectations', 'foo')

        cmd = ['trigger', '--flag=--foo']
        if regenerate:
            cmd.append('--regenerate')
        TryFlag(cmd, host, git_cl).run()

        expected_added_paths = {flag_file}
        expected_commits = [[
            'Flag try job: force --foo for run_web_tests.py.'
        ]]

        if regenerate:
            expected_added_paths.add(flag_expectations_file)
            expected_commits.append(
                ['Flag try job: clear expectations for --foo.'])

        self.assertEqual(git.added_paths, expected_added_paths)
        self.assertEqual(git.local_commits(), expected_commits)

        self.assertEqual(git_cl.calls, [[
            'git', 'cl', 'upload', '--bypass-hooks', '-f', '-m',
            'Flag try job for --foo.'
        ], [
            'git', 'cl', 'try', '-B', 'luci.chromium.try', '-b', 'linux-rel'
        ], [
            'git', 'cl', 'try', '-B', 'luci.chromium.try', '-b', 'mac-rel'
        ], ['git', 'cl', 'try', '-B', 'luci.chromium.try', '-b', 'win7-rel']])

    def test_trigger(self):
        self._run_trigger_test(regenerate=False)
        self._run_trigger_test(regenerate=True)

    def _setup_mock_results(self, results_fetcher):
        results_fetcher.set_results(
            self.linux_build,
            WebTestResults.from_json(
                {
                    'tests': {
                        'something': {
                            'fail-everywhere.html': {
                                'expected': 'FAIL',
                                'actual': 'FAIL',
                                'is_unexpected': True
                            },
                            'fail-win-and-linux.html': {
                                'expected': 'FAIL',
                                'actual': 'FAIL',
                                'is_unexpected': True
                            }
                        }
                    }
                },
                step_name='blink_web_tests'))
        results_fetcher.set_results(
            self.win_build,
            WebTestResults.from_json(
                {
                    'tests': {
                        'something': {
                            'fail-everywhere.html': {
                                'expected': 'FAIL',
                                'actual': 'FAIL',
                                'is_unexpected': True
                            },
                            'fail-win-and-linux.html': {
                                'expected': 'FAIL',
                                'actual': 'FAIL',
                                'is_unexpected': True
                            }
                        }
                    }
                },
                step_name='blink_web_tests'))
        results_fetcher.set_results(
            self.mac_build,
            WebTestResults.from_json(
                {
                    'tests': {
                        'something': {
                            'pass-unexpectedly-mac.html': {
                                'expected': 'FAIL',
                                'actual': 'PASS',
                                'is_unexpected': True
                            },
                            'fail-everywhere.html': {
                                'expected': 'FAIL',
                                'actual': 'FAIL',
                                'is_unexpected': True
                            }
                        }
                    }
                },
                step_name='blink_web_tests'))

    def test_update(self):
        host = MockHost()
        filesystem = host.filesystem
        finder = PathFinder(filesystem)

        flag_expectations_file = finder.path_from_web_tests(
            'FlagExpectations', 'foo')
        filesystem.write_text_file(
            flag_expectations_file,
            '# results: [ Failure ]\nsomething/pass-unexpectedly-mac.html [ Failure ]'
        )

        self._setup_mock_results(host.results_fetcher)
        cmd = ['update', '--flag=--foo']
        TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()

        self.assertEqual(
            host.stdout.getvalue(), '\n'.join([
                'Fetching results...', '', '### 1 unexpected passes:', '',
                '[ Mac ] something/pass-unexpectedly-mac.html [ Pass ]', '',
                '### 5 unexpected failures:', '',
                '[ Linux ] something/fail-everywhere.html [ Failure ]',
                '[ Mac ] something/fail-everywhere.html [ Failure ]',
                '[ Win ] something/fail-everywhere.html [ Failure ]',
                '[ Linux ] something/fail-win-and-linux.html [ Failure ]',
                '[ Win ] something/fail-win-and-linux.html [ Failure ]', ''
            ]))

    def test_update_irrelevant_unexpected_pass(self):
        host = MockHost()
        filesystem = host.filesystem
        finder = PathFinder(filesystem)
        flag_expectations_file = finder.path_from_web_tests(
            'FlagExpectations', 'foo')
        self._setup_mock_results(host.results_fetcher)
        cmd = ['update', '--flag=--foo']

        # Unexpected passes that don't have flag-specific failure expectations
        # should not be reported.
        filesystem.write_text_file(flag_expectations_file, '')
        TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()
        self.assertTrue('### 0 unexpected passes' in host.stdout.getvalue())

    def test_invalid_action(self):
        host = MockHost()
        cmd = ['invalid', '--flag=--foo']
        TryFlag(cmd, host, MockGitCL(host)).run()
        self.assertEqual(host.stderr.getvalue(),
                         'specify "trigger" or "update"\n')