OLD | NEW |
| (Empty) |
1 # Copyright (c) 2010 Google Inc. All rights reserved. | |
2 # | |
3 # Redistribution and use in source and binary forms, with or without | |
4 # modification, are permitted provided that the following conditions are | |
5 # met: | |
6 # | |
7 # * Redistributions of source code must retain the above copyright | |
8 # notice, this list of conditions and the following disclaimer. | |
9 # * Redistributions in binary form must reproduce the above | |
10 # copyright notice, this list of conditions and the following disclaimer | |
11 # in the documentation and/or other materials provided with the | |
12 # distribution. | |
13 # * Neither the name of Google Inc. nor the names of its | |
14 # contributors may be used to endorse or promote products derived from | |
15 # this software without specific prior written permission. | |
16 # | |
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
28 | |
29 from datetime import datetime | |
30 import logging | |
31 import unittest2 as unittest | |
32 | |
33 from webkitpy.common.net import bugzilla | |
34 from webkitpy.common.net.layouttestresults import LayoutTestResults | |
35 from webkitpy.common.system.executive import ScriptError | |
36 from webkitpy.common.system.outputcapture import OutputCapture | |
37 from webkitpy.layout_tests.models import test_results | |
38 from webkitpy.layout_tests.models import test_failures | |
39 from webkitpy.thirdparty.mock import Mock | |
40 from webkitpy.tool.bot.commitqueuetask import * | |
41 from webkitpy.tool.bot.expectedfailures import ExpectedFailures | |
42 from webkitpy.tool.mocktool import MockTool | |
43 | |
44 _log = logging.getLogger(__name__) | |
45 | |
46 | |
47 class MockCommitQueue(CommitQueueTaskDelegate): | |
48 def __init__(self, error_plan): | |
49 self._error_plan = error_plan | |
50 self._failure_status_id = 0 | |
51 | |
52 def run_command(self, command): | |
53 _log.info("run_webkit_patch: %s" % command) | |
54 if self._error_plan: | |
55 error = self._error_plan.pop(0) | |
56 if error: | |
57 raise error | |
58 | |
59 def command_passed(self, success_message, patch): | |
60 _log.info("command_passed: success_message='%s' patch='%s'" % ( | |
61 success_message, patch.id())) | |
62 | |
63 def command_failed(self, failure_message, script_error, patch): | |
64 _log.info("command_failed: failure_message='%s' script_error='%s' patch=
'%s'" % ( | |
65 failure_message, script_error, patch.id())) | |
66 self._failure_status_id += 1 | |
67 return self._failure_status_id | |
68 | |
69 def refetch_patch(self, patch): | |
70 return patch | |
71 | |
72 def expected_failures(self): | |
73 return ExpectedFailures() | |
74 | |
75 def test_results(self): | |
76 return None | |
77 | |
78 def report_flaky_tests(self, patch, flaky_results, results_archive): | |
79 flaky_tests = [result.filename for result in flaky_results] | |
80 _log.info("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'"
% (patch.id(), flaky_tests, results_archive.filename)) | |
81 | |
82 def archive_last_test_results(self, patch): | |
83 _log.info("archive_last_test_results: patch='%s'" % patch.id()) | |
84 archive = Mock() | |
85 archive.filename = "mock-archive-%s.zip" % patch.id() | |
86 return archive | |
87 | |
88 def build_style(self): | |
89 return "both" | |
90 | |
91 def did_pass_testing_ews(self, patch): | |
92 return False | |
93 | |
94 | |
95 class FailingTestCommitQueue(MockCommitQueue): | |
96 def __init__(self, error_plan, test_failure_plan): | |
97 MockCommitQueue.__init__(self, error_plan) | |
98 self._test_run_counter = -1 # Special value to indicate tests have neve
r been run. | |
99 self._test_failure_plan = test_failure_plan | |
100 | |
101 def run_command(self, command): | |
102 if command[0] == "build-and-test": | |
103 self._test_run_counter += 1 | |
104 MockCommitQueue.run_command(self, command) | |
105 | |
106 def _mock_test_result(self, testname): | |
107 return test_results.TestResult(testname, [test_failures.FailureTextMisma
tch()]) | |
108 | |
109 def test_results(self): | |
110 # Doesn't make sense to ask for the test_results until the tests have ru
n at least once. | |
111 assert(self._test_run_counter >= 0) | |
112 failures_for_run = self._test_failure_plan[self._test_run_counter] | |
113 results = LayoutTestResults(map(self._mock_test_result, failures_for_run
)) | |
114 # This makes the results trustable by ExpectedFailures. | |
115 results.set_failure_limit_count(10) | |
116 return results | |
117 | |
118 | |
119 # We use GoldenScriptError to make sure that the code under test throws the | |
120 # correct (i.e., golden) exception. | |
121 class GoldenScriptError(ScriptError): | |
122 pass | |
123 | |
124 | |
125 class CommitQueueTaskTest(unittest.TestCase): | |
126 def _run_through_task(self, commit_queue, expected_logs, expected_exception=
None, expect_retry=False): | |
127 tool = MockTool(log_executive=True) | |
128 patch = tool.bugs.fetch_attachment(10000) | |
129 task = CommitQueueTask(commit_queue, patch) | |
130 success = OutputCapture().assert_outputs(self, task.run, expected_logs=e
xpected_logs, expected_exception=expected_exception) | |
131 if not expected_exception: | |
132 self.assertEqual(success, not expect_retry) | |
133 return task | |
134 | |
135 def test_success_case(self): | |
136 commit_queue = MockCommitQueue([]) | |
137 expected_logs = """run_webkit_patch: ['clean'] | |
138 command_passed: success_message='Cleaned working directory' patch='10000' | |
139 run_webkit_patch: ['update'] | |
140 command_passed: success_message='Updated working directory' patch='10000' | |
141 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
142 command_passed: success_message='Applied patch' patch='10000' | |
143 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
144 command_passed: success_message='ChangeLog validated' patch='10000' | |
145 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
146 command_passed: success_message='Built patch' patch='10000' | |
147 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
148 command_passed: success_message='Passed tests' patch='10000' | |
149 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
150 command_passed: success_message='Landed patch' patch='10000' | |
151 """ | |
152 self._run_through_task(commit_queue, expected_logs) | |
153 | |
154 def test_fast_success_case(self): | |
155 commit_queue = MockCommitQueue([]) | |
156 commit_queue.did_pass_testing_ews = lambda patch: True | |
157 expected_logs = """run_webkit_patch: ['clean'] | |
158 command_passed: success_message='Cleaned working directory' patch='10000' | |
159 run_webkit_patch: ['update'] | |
160 command_passed: success_message='Updated working directory' patch='10000' | |
161 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
162 command_passed: success_message='Applied patch' patch='10000' | |
163 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
164 command_passed: success_message='ChangeLog validated' patch='10000' | |
165 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
166 command_passed: success_message='Built patch' patch='10000' | |
167 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
168 command_passed: success_message='Landed patch' patch='10000' | |
169 """ | |
170 self._run_through_task(commit_queue, expected_logs) | |
171 | |
172 def test_clean_failure(self): | |
173 commit_queue = MockCommitQueue([ | |
174 ScriptError("MOCK clean failure"), | |
175 ]) | |
176 expected_logs = """run_webkit_patch: ['clean'] | |
177 command_failed: failure_message='Unable to clean working directory' script_error
='MOCK clean failure' patch='10000' | |
178 """ | |
179 self._run_through_task(commit_queue, expected_logs, expect_retry=True) | |
180 | |
181 def test_update_failure(self): | |
182 commit_queue = MockCommitQueue([ | |
183 None, | |
184 ScriptError("MOCK update failure"), | |
185 ]) | |
186 expected_logs = """run_webkit_patch: ['clean'] | |
187 command_passed: success_message='Cleaned working directory' patch='10000' | |
188 run_webkit_patch: ['update'] | |
189 command_failed: failure_message='Unable to update working directory' script_erro
r='MOCK update failure' patch='10000' | |
190 """ | |
191 self._run_through_task(commit_queue, expected_logs, expect_retry=True) | |
192 | |
193 def test_apply_failure(self): | |
194 commit_queue = MockCommitQueue([ | |
195 None, | |
196 None, | |
197 GoldenScriptError("MOCK apply failure"), | |
198 ]) | |
199 expected_logs = """run_webkit_patch: ['clean'] | |
200 command_passed: success_message='Cleaned working directory' patch='10000' | |
201 run_webkit_patch: ['update'] | |
202 command_passed: success_message='Updated working directory' patch='10000' | |
203 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
204 command_failed: failure_message='Patch does not apply' script_error='MOCK apply
failure' patch='10000' | |
205 """ | |
206 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
207 | |
208 def test_validate_changelog_failure(self): | |
209 commit_queue = MockCommitQueue([ | |
210 None, | |
211 None, | |
212 None, | |
213 GoldenScriptError("MOCK validate failure"), | |
214 ]) | |
215 expected_logs = """run_webkit_patch: ['clean'] | |
216 command_passed: success_message='Cleaned working directory' patch='10000' | |
217 run_webkit_patch: ['update'] | |
218 command_passed: success_message='Updated working directory' patch='10000' | |
219 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
220 command_passed: success_message='Applied patch' patch='10000' | |
221 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
222 command_failed: failure_message='ChangeLog did not pass validation' script_error
='MOCK validate failure' patch='10000' | |
223 """ | |
224 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
225 | |
226 def test_build_failure(self): | |
227 commit_queue = MockCommitQueue([ | |
228 None, | |
229 None, | |
230 None, | |
231 None, | |
232 GoldenScriptError("MOCK build failure"), | |
233 ]) | |
234 expected_logs = """run_webkit_patch: ['clean'] | |
235 command_passed: success_message='Cleaned working directory' patch='10000' | |
236 run_webkit_patch: ['update'] | |
237 command_passed: success_message='Updated working directory' patch='10000' | |
238 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
239 command_passed: success_message='Applied patch' patch='10000' | |
240 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
241 command_passed: success_message='ChangeLog validated' patch='10000' | |
242 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
243 command_failed: failure_message='Patch does not build' script_error='MOCK build
failure' patch='10000' | |
244 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both'
] | |
245 command_passed: success_message='Able to build without patch' patch='10000' | |
246 """ | |
247 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
248 | |
249 def test_red_build_failure(self): | |
250 commit_queue = MockCommitQueue([ | |
251 None, | |
252 None, | |
253 None, | |
254 None, | |
255 ScriptError("MOCK build failure"), | |
256 ScriptError("MOCK clean build failure"), | |
257 ]) | |
258 expected_logs = """run_webkit_patch: ['clean'] | |
259 command_passed: success_message='Cleaned working directory' patch='10000' | |
260 run_webkit_patch: ['update'] | |
261 command_passed: success_message='Updated working directory' patch='10000' | |
262 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
263 command_passed: success_message='Applied patch' patch='10000' | |
264 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
265 command_passed: success_message='ChangeLog validated' patch='10000' | |
266 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
267 command_failed: failure_message='Patch does not build' script_error='MOCK build
failure' patch='10000' | |
268 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both'
] | |
269 command_failed: failure_message='Unable to build without patch' script_error='MO
CK clean build failure' patch='10000' | |
270 """ | |
271 self._run_through_task(commit_queue, expected_logs, expect_retry=True) | |
272 | |
273 def test_flaky_test_failure(self): | |
274 commit_queue = MockCommitQueue([ | |
275 None, | |
276 None, | |
277 None, | |
278 None, | |
279 None, | |
280 ScriptError("MOCK tests failure"), | |
281 ]) | |
282 # CommitQueueTask will only report flaky tests if we successfully parsed | |
283 # results.json and returned a LayoutTestResults object, so we fake one. | |
284 commit_queue.test_results = lambda: LayoutTestResults([]) | |
285 expected_logs = """run_webkit_patch: ['clean'] | |
286 command_passed: success_message='Cleaned working directory' patch='10000' | |
287 run_webkit_patch: ['update'] | |
288 command_passed: success_message='Updated working directory' patch='10000' | |
289 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
290 command_passed: success_message='Applied patch' patch='10000' | |
291 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
292 command_passed: success_message='ChangeLog validated' patch='10000' | |
293 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
294 command_passed: success_message='Built patch' patch='10000' | |
295 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
296 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
ests failure' patch='10000' | |
297 archive_last_test_results: patch='10000' | |
298 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
299 command_passed: success_message='Passed tests' patch='10000' | |
300 report_flaky_tests: patch='10000' flaky_tests='[]' archive='mock-archive-10000.z
ip' | |
301 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
302 command_passed: success_message='Landed patch' patch='10000' | |
303 """ | |
304 self._run_through_task(commit_queue, expected_logs) | |
305 | |
306 def test_failed_archive(self): | |
307 commit_queue = MockCommitQueue([ | |
308 None, | |
309 None, | |
310 None, | |
311 None, | |
312 None, | |
313 ScriptError("MOCK tests failure"), | |
314 ]) | |
315 commit_queue.test_results = lambda: LayoutTestResults([]) | |
316 # It's possible delegate to fail to archive layout tests, don't try to r
eport | |
317 # flaky tests when that happens. | |
318 commit_queue.archive_last_test_results = lambda patch: None | |
319 expected_logs = """run_webkit_patch: ['clean'] | |
320 command_passed: success_message='Cleaned working directory' patch='10000' | |
321 run_webkit_patch: ['update'] | |
322 command_passed: success_message='Updated working directory' patch='10000' | |
323 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
324 command_passed: success_message='Applied patch' patch='10000' | |
325 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
326 command_passed: success_message='ChangeLog validated' patch='10000' | |
327 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
328 command_passed: success_message='Built patch' patch='10000' | |
329 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
330 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
ests failure' patch='10000' | |
331 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
332 command_passed: success_message='Passed tests' patch='10000' | |
333 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
334 command_passed: success_message='Landed patch' patch='10000' | |
335 """ | |
336 self._run_through_task(commit_queue, expected_logs) | |
337 | |
338 def test_double_flaky_test_failure(self): | |
339 commit_queue = FailingTestCommitQueue([ | |
340 None, | |
341 None, | |
342 None, | |
343 None, | |
344 None, | |
345 ScriptError("MOCK test failure"), | |
346 ScriptError("MOCK test failure again"), | |
347 ], [ | |
348 "foo.html", | |
349 "bar.html", | |
350 "foo.html", | |
351 ]) | |
352 # The (subtle) point of this test is that report_flaky_tests does not ap
pear | |
353 # in the expected_logs for this run. | |
354 # Note also that there is no attempt to run the tests w/o the patch. | |
355 expected_logs = """run_webkit_patch: ['clean'] | |
356 command_passed: success_message='Cleaned working directory' patch='10000' | |
357 run_webkit_patch: ['update'] | |
358 command_passed: success_message='Updated working directory' patch='10000' | |
359 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
360 command_passed: success_message='Applied patch' patch='10000' | |
361 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
362 command_passed: success_message='ChangeLog validated' patch='10000' | |
363 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
364 command_passed: success_message='Built patch' patch='10000' | |
365 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
366 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
367 archive_last_test_results: patch='10000' | |
368 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
369 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
370 """ | |
371 tool = MockTool(log_executive=True) | |
372 patch = tool.bugs.fetch_attachment(10000) | |
373 task = CommitQueueTask(commit_queue, patch) | |
374 success = OutputCapture().assert_outputs(self, task.run, expected_logs=e
xpected_logs) | |
375 self.assertFalse(success) | |
376 | |
377 def test_test_failure(self): | |
378 commit_queue = MockCommitQueue([ | |
379 None, | |
380 None, | |
381 None, | |
382 None, | |
383 None, | |
384 GoldenScriptError("MOCK test failure"), | |
385 ScriptError("MOCK test failure again"), | |
386 ]) | |
387 expected_logs = """run_webkit_patch: ['clean'] | |
388 command_passed: success_message='Cleaned working directory' patch='10000' | |
389 run_webkit_patch: ['update'] | |
390 command_passed: success_message='Updated working directory' patch='10000' | |
391 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
392 command_passed: success_message='Applied patch' patch='10000' | |
393 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
394 command_passed: success_message='ChangeLog validated' patch='10000' | |
395 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
396 command_passed: success_message='Built patch' patch='10000' | |
397 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
398 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
399 archive_last_test_results: patch='10000' | |
400 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
401 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
402 archive_last_test_results: patch='10000' | |
403 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build',
'--test', '--non-interactive'] | |
404 command_passed: success_message='Able to pass tests without patch' patch='10000' | |
405 """ | |
406 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
407 | |
408 def test_red_test_failure(self): | |
409 commit_queue = FailingTestCommitQueue([ | |
410 None, | |
411 None, | |
412 None, | |
413 None, | |
414 None, | |
415 ScriptError("MOCK test failure"), | |
416 ScriptError("MOCK test failure again"), | |
417 ScriptError("MOCK clean test failure"), | |
418 ], [ | |
419 "foo.html", | |
420 "foo.html", | |
421 "foo.html", | |
422 ]) | |
423 | |
424 # Tests always fail, and always return the same results, but we | |
425 # should still be able to land in this case! | |
426 expected_logs = """run_webkit_patch: ['clean'] | |
427 command_passed: success_message='Cleaned working directory' patch='10000' | |
428 run_webkit_patch: ['update'] | |
429 command_passed: success_message='Updated working directory' patch='10000' | |
430 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
431 command_passed: success_message='Applied patch' patch='10000' | |
432 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
433 command_passed: success_message='ChangeLog validated' patch='10000' | |
434 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
435 command_passed: success_message='Built patch' patch='10000' | |
436 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
437 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
438 archive_last_test_results: patch='10000' | |
439 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
440 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
441 archive_last_test_results: patch='10000' | |
442 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build',
'--test', '--non-interactive'] | |
443 command_failed: failure_message='Unable to pass tests without patch (tree is red
?)' script_error='MOCK clean test failure' patch='10000' | |
444 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
445 command_passed: success_message='Landed patch' patch='10000' | |
446 """ | |
447 self._run_through_task(commit_queue, expected_logs) | |
448 | |
449 def test_very_red_tree_retry(self): | |
450 lots_of_failing_tests = map(lambda num: "test-%s.html" % num, range(0, 1
00)) | |
451 commit_queue = FailingTestCommitQueue([ | |
452 None, | |
453 None, | |
454 None, | |
455 None, | |
456 None, | |
457 ScriptError("MOCK test failure"), | |
458 ScriptError("MOCK test failure again"), | |
459 ScriptError("MOCK clean test failure"), | |
460 ], [ | |
461 lots_of_failing_tests, | |
462 lots_of_failing_tests, | |
463 lots_of_failing_tests, | |
464 ]) | |
465 | |
466 # Tests always fail, and return so many failures that we do not | |
467 # trust the results (see ExpectedFailures._can_trust_results) so we | |
468 # just give up and retry the patch. | |
469 expected_logs = """run_webkit_patch: ['clean'] | |
470 command_passed: success_message='Cleaned working directory' patch='10000' | |
471 run_webkit_patch: ['update'] | |
472 command_passed: success_message='Updated working directory' patch='10000' | |
473 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
474 command_passed: success_message='Applied patch' patch='10000' | |
475 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
476 command_passed: success_message='ChangeLog validated' patch='10000' | |
477 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
478 command_passed: success_message='Built patch' patch='10000' | |
479 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
480 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
481 archive_last_test_results: patch='10000' | |
482 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
483 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
484 archive_last_test_results: patch='10000' | |
485 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build',
'--test', '--non-interactive'] | |
486 command_failed: failure_message='Unable to pass tests without patch (tree is red
?)' script_error='MOCK clean test failure' patch='10000' | |
487 """ | |
488 self._run_through_task(commit_queue, expected_logs, expect_retry=True) | |
489 | |
490 def test_red_tree_patch_rejection(self): | |
491 commit_queue = FailingTestCommitQueue([ | |
492 None, | |
493 None, | |
494 None, | |
495 None, | |
496 None, | |
497 GoldenScriptError("MOCK test failure"), | |
498 ScriptError("MOCK test failure again"), | |
499 ScriptError("MOCK clean test failure"), | |
500 ], [ | |
501 ["foo.html", "bar.html"], | |
502 ["foo.html", "bar.html"], | |
503 ["foo.html"], | |
504 ]) | |
505 | |
506 # Tests always fail, but the clean tree only fails one test | |
507 # while the patch fails two. So we should reject the patch! | |
508 expected_logs = """run_webkit_patch: ['clean'] | |
509 command_passed: success_message='Cleaned working directory' patch='10000' | |
510 run_webkit_patch: ['update'] | |
511 command_passed: success_message='Updated working directory' patch='10000' | |
512 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
513 command_passed: success_message='Applied patch' patch='10000' | |
514 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
515 command_passed: success_message='ChangeLog validated' patch='10000' | |
516 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
517 command_passed: success_message='Built patch' patch='10000' | |
518 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
519 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
520 archive_last_test_results: patch='10000' | |
521 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
522 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
523 archive_last_test_results: patch='10000' | |
524 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build',
'--test', '--non-interactive'] | |
525 command_failed: failure_message='Unable to pass tests without patch (tree is red
?)' script_error='MOCK clean test failure' patch='10000' | |
526 """ | |
527 task = self._run_through_task(commit_queue, expected_logs, GoldenScriptE
rror) | |
528 self.assertEqual(task.results_from_patch_test_run(task._patch).failing_t
ests(), ["foo.html", "bar.html"]) | |
529 # failure_status_id should be of the test with patch (1), not the test w
ithout patch (2). | |
530 self.assertEqual(task.failure_status_id, 1) | |
531 | |
532 def test_land_failure(self): | |
533 commit_queue = MockCommitQueue([ | |
534 None, | |
535 None, | |
536 None, | |
537 None, | |
538 None, | |
539 None, | |
540 GoldenScriptError("MOCK land failure"), | |
541 ]) | |
542 expected_logs = """run_webkit_patch: ['clean'] | |
543 command_passed: success_message='Cleaned working directory' patch='10000' | |
544 run_webkit_patch: ['update'] | |
545 command_passed: success_message='Updated working directory' patch='10000' | |
546 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
547 command_passed: success_message='Applied patch' patch='10000' | |
548 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
549 command_passed: success_message='ChangeLog validated' patch='10000' | |
550 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
551 command_passed: success_message='Built patch' patch='10000' | |
552 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
553 command_passed: success_message='Passed tests' patch='10000' | |
554 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
555 command_failed: failure_message='Unable to land patch' script_error='MOCK land f
ailure' patch='10000' | |
556 """ | |
557 # FIXME: This should really be expect_retry=True for a better user exper
iance. | |
558 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
559 | |
560 def _expect_validate(self, patch, is_valid): | |
561 class MockDelegate(object): | |
562 def refetch_patch(self, patch): | |
563 return patch | |
564 | |
565 def expected_failures(self): | |
566 return ExpectedFailures() | |
567 | |
568 task = CommitQueueTask(MockDelegate(), patch) | |
569 self.assertEqual(task.validate(), is_valid) | |
570 | |
571 def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, co
mmitter="fake"): | |
572 bug = bugzilla.Bug(bug_dict, None) | |
573 patch = bugzilla.Attachment(attachment_dict, bug) | |
574 patch._committer = committer | |
575 return patch | |
576 | |
577 def test_validate(self): | |
578 self._expect_validate(self._mock_patch(), True) | |
579 self._expect_validate(self._mock_patch({'is_obsolete': True}), False) | |
580 self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}
), False) | |
581 self._expect_validate(self._mock_patch(committer=None), False) | |
582 self._expect_validate(self._mock_patch({'review': '-'}), False) | |
OLD | NEW |