OLD | NEW |
| (Empty) |
1 # Copyright (c) 2010 Google Inc. All rights reserved. | |
2 # | |
3 # Redistribution and use in source and binary forms, with or without | |
4 # modification, are permitted provided that the following conditions are | |
5 # met: | |
6 # | |
7 # * Redistributions of source code must retain the above copyright | |
8 # notice, this list of conditions and the following disclaimer. | |
9 # * Redistributions in binary form must reproduce the above | |
10 # copyright notice, this list of conditions and the following disclaimer | |
11 # in the documentation and/or other materials provided with the | |
12 # distribution. | |
13 # * Neither the name of Google Inc. nor the names of its | |
14 # contributors may be used to endorse or promote products derived from | |
15 # this software without specific prior written permission. | |
16 # | |
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
28 | |
29 import unittest2 as unittest | |
30 | |
31 from webkitpy.common.config.committers import Committer | |
32 from webkitpy.common.system.filesystem_mock import MockFileSystem | |
33 from webkitpy.common.system.outputcapture import OutputCapture | |
34 from webkitpy.layout_tests.models import test_results | |
35 from webkitpy.layout_tests.models import test_failures | |
36 from webkitpy.thirdparty.mock import Mock | |
37 from webkitpy.tool.bot.flakytestreporter import FlakyTestReporter | |
38 from webkitpy.tool.mocktool import MockTool | |
39 from webkitpy.common.net.statusserver_mock import MockStatusServer | |
40 | |
41 | |
42 # Creating fake CommitInfos is a pain, so we use a mock one here. | |
43 class MockCommitInfo(object): | |
44 def __init__(self, author_email): | |
45 self._author_email = author_email | |
46 | |
47 def author(self): | |
48 # It's definitely possible to have commits with authors who | |
49 # are not in our committers.py list. | |
50 if not self._author_email: | |
51 return None | |
52 return Committer("Mock Committer", self._author_email) | |
53 | |
54 | |
55 class FlakyTestReporterTest(unittest.TestCase): | |
56 def _mock_test_result(self, testname): | |
57 return test_results.TestResult(testname, [test_failures.FailureTextMisma
tch()]) | |
58 | |
59 def _assert_emails_for_test(self, emails): | |
60 tool = MockTool() | |
61 reporter = FlakyTestReporter(tool, 'dummy-queue') | |
62 commit_infos = [MockCommitInfo(email) for email in emails] | |
63 tool.checkout().recent_commit_infos_for_files = lambda paths: set(commit
_infos) | |
64 self.assertEqual(reporter._author_emails_for_test([]), set(emails)) | |
65 | |
66 def test_author_emails_for_test(self): | |
67 self._assert_emails_for_test([]) | |
68 self._assert_emails_for_test(["test1@test.com", "test1@test.com"]) | |
69 self._assert_emails_for_test(["test1@test.com", "test2@test.com"]) | |
70 | |
71 def test_create_bug_for_flaky_test(self): | |
72 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') | |
73 expected_logs = """MOCK create_bug | |
74 bug_title: Flaky Test: foo/bar.html | |
75 bug_description: This is an automatically generated bug from the dummy-queue. | |
76 foo/bar.html has been flaky on the dummy-queue. | |
77 | |
78 foo/bar.html was authored by test@test.com. | |
79 http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html | |
80 | |
81 FLAKE_MESSAGE | |
82 | |
83 The bots will update this with information from each new failure. | |
84 | |
85 If you believe this bug to be fixed or invalid, feel free to close. The bots wi
ll re-open if the flake re-occurs. | |
86 | |
87 If you would like to track this test fix with another bug, please close this bug
as a duplicate. The bots will follow the duplicate chain when making future co
mments. | |
88 | |
89 component: Tools / Tests | |
90 cc: test@test.com | |
91 blocked: 50856 | |
92 """ | |
93 OutputCapture().assert_outputs(self, reporter._create_bug_for_flaky_test
, ['foo/bar.html', ['test@test.com'], 'FLAKE_MESSAGE'], expected_logs=expected_l
ogs) | |
94 | |
95 def test_follow_duplicate_chain(self): | |
96 tool = MockTool() | |
97 reporter = FlakyTestReporter(tool, 'dummy-queue') | |
98 bug = tool.bugs.fetch_bug(50004) | |
99 self.assertEqual(reporter._follow_duplicate_chain(bug).id(), 50002) | |
100 | |
101 def test_report_flaky_tests_creating_bug(self): | |
102 tool = MockTool() | |
103 tool.filesystem = MockFileSystem({"/mock-results/foo/bar-diffs.txt": "mo
ck"}) | |
104 tool.status_server = MockStatusServer(bot_id="mock-bot-id") | |
105 reporter = FlakyTestReporter(tool, 'dummy-queue') | |
106 reporter._lookup_bug_for_flaky_test = lambda bug_id: None | |
107 patch = tool.bugs.fetch_attachment(10000) | |
108 expected_logs = """Bug does not already exist for foo/bar.html, creating
. | |
109 MOCK create_bug | |
110 bug_title: Flaky Test: foo/bar.html | |
111 bug_description: This is an automatically generated bug from the dummy-queue. | |
112 foo/bar.html has been flaky on the dummy-queue. | |
113 | |
114 foo/bar.html was authored by abarth@webkit.org. | |
115 http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html | |
116 | |
117 The dummy-queue just saw foo/bar.html flake (text diff) while processing attachm
ent 10000 on bug 50000. | |
118 Bot: mock-bot-id Port: MockPort Platform: MockPlatform 1.0 | |
119 | |
120 The bots will update this with information from each new failure. | |
121 | |
122 If you believe this bug to be fixed or invalid, feel free to close. The bots wi
ll re-open if the flake re-occurs. | |
123 | |
124 If you would like to track this test fix with another bug, please close this bug
as a duplicate. The bots will follow the duplicate chain when making future co
mments. | |
125 | |
126 component: Tools / Tests | |
127 cc: abarth@webkit.org | |
128 blocked: 50856 | |
129 MOCK add_attachment_to_bug: bug_id=60001, description=Failure diff from mock-bot
-id filename=failure.diff mimetype=None | |
130 MOCK bug comment: bug_id=50000, cc=None | |
131 --- Begin comment --- | |
132 The dummy-queue encountered the following flaky tests while processing attachmen
t 10000: | |
133 | |
134 foo/bar.html bug 60001 (author: abarth@webkit.org) | |
135 The dummy-queue is continuing to process your patch. | |
136 --- End comment --- | |
137 | |
138 """ | |
139 test_results = [self._mock_test_result('foo/bar.html')] | |
140 | |
141 class MockZipFile(object): | |
142 def read(self, path): | |
143 return "" | |
144 | |
145 def namelist(self): | |
146 return ['foo/bar-diffs.txt'] | |
147 | |
148 OutputCapture().assert_outputs(self, reporter.report_flaky_tests, [patch
, test_results, MockZipFile()], expected_logs=expected_logs) | |
149 | |
150 def test_optional_author_string(self): | |
151 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') | |
152 self.assertEqual(reporter._optional_author_string([]), "") | |
153 self.assertEqual(reporter._optional_author_string(["foo@bar.com"]), " (a
uthor: foo@bar.com)") | |
154 self.assertEqual(reporter._optional_author_string(["a@b.com", "b@b.com"]
), " (authors: a@b.com and b@b.com)") | |
155 | |
156 def test_results_diff_path_for_test(self): | |
157 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') | |
158 self.assertEqual(reporter._results_diff_path_for_test("test.html"), "tes
t-diffs.txt") | |
159 | |
160 def test_find_in_archive(self): | |
161 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') | |
162 | |
163 class MockZipFile(object): | |
164 def namelist(self): | |
165 return ["tmp/layout-test-results/foo/bar-diffs.txt"] | |
166 | |
167 reporter._find_in_archive("foo/bar-diffs.txt", MockZipFile()) | |
168 # This is not ideal, but its | |
169 reporter._find_in_archive("txt", MockZipFile()) | |
OLD | NEW |