| OLD | NEW |
| (Empty) |
| 1 # Copyright (c) 2010 Google Inc. All rights reserved. | |
| 2 # | |
| 3 # Redistribution and use in source and binary forms, with or without | |
| 4 # modification, are permitted provided that the following conditions are | |
| 5 # met: | |
| 6 # | |
| 7 # * Redistributions of source code must retain the above copyright | |
| 8 # notice, this list of conditions and the following disclaimer. | |
| 9 # * Redistributions in binary form must reproduce the above | |
| 10 # copyright notice, this list of conditions and the following disclaimer | |
| 11 # in the documentation and/or other materials provided with the | |
| 12 # distribution. | |
| 13 # * Neither the name of Google Inc. nor the names of its | |
| 14 # contributors may be used to endorse or promote products derived from | |
| 15 # this software without specific prior written permission. | |
| 16 # | |
| 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 28 | |
| 29 import unittest2 as unittest | |
| 30 | |
| 31 from webkitpy.common.config.committers import Committer | |
| 32 from webkitpy.common.system.filesystem_mock import MockFileSystem | |
| 33 from webkitpy.common.system.outputcapture import OutputCapture | |
| 34 from webkitpy.layout_tests.models import test_results | |
| 35 from webkitpy.layout_tests.models import test_failures | |
| 36 from webkitpy.thirdparty.mock import Mock | |
| 37 from webkitpy.tool.bot.flakytestreporter import FlakyTestReporter | |
| 38 from webkitpy.tool.mocktool import MockTool | |
| 39 from webkitpy.common.net.statusserver_mock import MockStatusServer | |
| 40 | |
| 41 | |
| 42 # Creating fake CommitInfos is a pain, so we use a mock one here. | |
| 43 class MockCommitInfo(object): | |
| 44 def __init__(self, author_email): | |
| 45 self._author_email = author_email | |
| 46 | |
| 47 def author(self): | |
| 48 # It's definitely possible to have commits with authors who | |
| 49 # are not in our committers.py list. | |
| 50 if not self._author_email: | |
| 51 return None | |
| 52 return Committer("Mock Committer", self._author_email) | |
| 53 | |
| 54 | |
| 55 class FlakyTestReporterTest(unittest.TestCase): | |
| 56 def _mock_test_result(self, testname): | |
| 57 return test_results.TestResult(testname, [test_failures.FailureTextMisma
tch()]) | |
| 58 | |
| 59 def _assert_emails_for_test(self, emails): | |
| 60 tool = MockTool() | |
| 61 reporter = FlakyTestReporter(tool, 'dummy-queue') | |
| 62 commit_infos = [MockCommitInfo(email) for email in emails] | |
| 63 tool.checkout().recent_commit_infos_for_files = lambda paths: set(commit
_infos) | |
| 64 self.assertEqual(reporter._author_emails_for_test([]), set(emails)) | |
| 65 | |
| 66 def test_author_emails_for_test(self): | |
| 67 self._assert_emails_for_test([]) | |
| 68 self._assert_emails_for_test(["test1@test.com", "test1@test.com"]) | |
| 69 self._assert_emails_for_test(["test1@test.com", "test2@test.com"]) | |
| 70 | |
| 71 def test_create_bug_for_flaky_test(self): | |
| 72 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') | |
| 73 expected_logs = """MOCK create_bug | |
| 74 bug_title: Flaky Test: foo/bar.html | |
| 75 bug_description: This is an automatically generated bug from the dummy-queue. | |
| 76 foo/bar.html has been flaky on the dummy-queue. | |
| 77 | |
| 78 foo/bar.html was authored by test@test.com. | |
| 79 http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html | |
| 80 | |
| 81 FLAKE_MESSAGE | |
| 82 | |
| 83 The bots will update this with information from each new failure. | |
| 84 | |
| 85 If you believe this bug to be fixed or invalid, feel free to close. The bots wi
ll re-open if the flake re-occurs. | |
| 86 | |
| 87 If you would like to track this test fix with another bug, please close this bug
as a duplicate. The bots will follow the duplicate chain when making future co
mments. | |
| 88 | |
| 89 component: Tools / Tests | |
| 90 cc: test@test.com | |
| 91 blocked: 50856 | |
| 92 """ | |
| 93 OutputCapture().assert_outputs(self, reporter._create_bug_for_flaky_test
, ['foo/bar.html', ['test@test.com'], 'FLAKE_MESSAGE'], expected_logs=expected_l
ogs) | |
| 94 | |
| 95 def test_follow_duplicate_chain(self): | |
| 96 tool = MockTool() | |
| 97 reporter = FlakyTestReporter(tool, 'dummy-queue') | |
| 98 bug = tool.bugs.fetch_bug(50004) | |
| 99 self.assertEqual(reporter._follow_duplicate_chain(bug).id(), 50002) | |
| 100 | |
| 101 def test_report_flaky_tests_creating_bug(self): | |
| 102 tool = MockTool() | |
| 103 tool.filesystem = MockFileSystem({"/mock-results/foo/bar-diffs.txt": "mo
ck"}) | |
| 104 tool.status_server = MockStatusServer(bot_id="mock-bot-id") | |
| 105 reporter = FlakyTestReporter(tool, 'dummy-queue') | |
| 106 reporter._lookup_bug_for_flaky_test = lambda bug_id: None | |
| 107 patch = tool.bugs.fetch_attachment(10000) | |
| 108 expected_logs = """Bug does not already exist for foo/bar.html, creating
. | |
| 109 MOCK create_bug | |
| 110 bug_title: Flaky Test: foo/bar.html | |
| 111 bug_description: This is an automatically generated bug from the dummy-queue. | |
| 112 foo/bar.html has been flaky on the dummy-queue. | |
| 113 | |
| 114 foo/bar.html was authored by abarth@webkit.org. | |
| 115 http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html | |
| 116 | |
| 117 The dummy-queue just saw foo/bar.html flake (text diff) while processing attachm
ent 10000 on bug 50000. | |
| 118 Bot: mock-bot-id Port: MockPort Platform: MockPlatform 1.0 | |
| 119 | |
| 120 The bots will update this with information from each new failure. | |
| 121 | |
| 122 If you believe this bug to be fixed or invalid, feel free to close. The bots wi
ll re-open if the flake re-occurs. | |
| 123 | |
| 124 If you would like to track this test fix with another bug, please close this bug
as a duplicate. The bots will follow the duplicate chain when making future co
mments. | |
| 125 | |
| 126 component: Tools / Tests | |
| 127 cc: abarth@webkit.org | |
| 128 blocked: 50856 | |
| 129 MOCK add_attachment_to_bug: bug_id=60001, description=Failure diff from mock-bot
-id filename=failure.diff mimetype=None | |
| 130 MOCK bug comment: bug_id=50000, cc=None | |
| 131 --- Begin comment --- | |
| 132 The dummy-queue encountered the following flaky tests while processing attachmen
t 10000: | |
| 133 | |
| 134 foo/bar.html bug 60001 (author: abarth@webkit.org) | |
| 135 The dummy-queue is continuing to process your patch. | |
| 136 --- End comment --- | |
| 137 | |
| 138 """ | |
| 139 test_results = [self._mock_test_result('foo/bar.html')] | |
| 140 | |
| 141 class MockZipFile(object): | |
| 142 def read(self, path): | |
| 143 return "" | |
| 144 | |
| 145 def namelist(self): | |
| 146 return ['foo/bar-diffs.txt'] | |
| 147 | |
| 148 OutputCapture().assert_outputs(self, reporter.report_flaky_tests, [patch
, test_results, MockZipFile()], expected_logs=expected_logs) | |
| 149 | |
| 150 def test_optional_author_string(self): | |
| 151 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') | |
| 152 self.assertEqual(reporter._optional_author_string([]), "") | |
| 153 self.assertEqual(reporter._optional_author_string(["foo@bar.com"]), " (a
uthor: foo@bar.com)") | |
| 154 self.assertEqual(reporter._optional_author_string(["a@b.com", "b@b.com"]
), " (authors: a@b.com and b@b.com)") | |
| 155 | |
| 156 def test_results_diff_path_for_test(self): | |
| 157 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') | |
| 158 self.assertEqual(reporter._results_diff_path_for_test("test.html"), "tes
t-diffs.txt") | |
| 159 | |
| 160 def test_find_in_archive(self): | |
| 161 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') | |
| 162 | |
| 163 class MockZipFile(object): | |
| 164 def namelist(self): | |
| 165 return ["tmp/layout-test-results/foo/bar-diffs.txt"] | |
| 166 | |
| 167 reporter._find_in_archive("foo/bar-diffs.txt", MockZipFile()) | |
| 168 # This is not ideal, but its | |
| 169 reporter._find_in_archive("txt", MockZipFile()) | |
| OLD | NEW |