| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2010 Google Inc. All rights reserved. | |
| 2 # | |
| 3 # Redistribution and use in source and binary forms, with or without | |
| 4 # modification, are permitted provided that the following conditions are | |
| 5 # met: | |
| 6 # | |
| 7 # * Redistributions of source code must retain the above copyright | |
| 8 # notice, this list of conditions and the following disclaimer. | |
| 9 # * Redistributions in binary form must reproduce the above | |
| 10 # copyright notice, this list of conditions and the following disclaimer | |
| 11 # in the documentation and/or other materials provided with the | |
| 12 # distribution. | |
| 13 # * Neither the name of Google Inc. nor the names of its | |
| 14 # contributors may be used to endorse or promote products derived from | |
| 15 # this software without specific prior written permission. | |
| 16 # | |
| 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 28 | |
| 29 from datetime import datetime | |
| 30 import logging | |
| 31 import unittest2 as unittest | |
| 32 | |
| 33 from webkitpy.common.net import bugzilla | |
| 34 from webkitpy.common.net.layouttestresults import LayoutTestResults | |
| 35 from webkitpy.common.system.executive import ScriptError | |
| 36 from webkitpy.common.system.outputcapture import OutputCapture | |
| 37 from webkitpy.layout_tests.models import test_results | |
| 38 from webkitpy.layout_tests.models import test_failures | |
| 39 from webkitpy.thirdparty.mock import Mock | |
| 40 from webkitpy.tool.bot.commitqueuetask import * | |
| 41 from webkitpy.tool.bot.expectedfailures import ExpectedFailures | |
| 42 from webkitpy.tool.mocktool import MockTool | |
| 43 | |
| 44 _log = logging.getLogger(__name__) | |
| 45 | |
| 46 | |
| 47 class MockCommitQueue(CommitQueueTaskDelegate): | |
| 48 def __init__(self, error_plan): | |
| 49 self._error_plan = error_plan | |
| 50 self._failure_status_id = 0 | |
| 51 | |
| 52 def run_command(self, command): | |
| 53 _log.info("run_webkit_patch: %s" % command) | |
| 54 if self._error_plan: | |
| 55 error = self._error_plan.pop(0) | |
| 56 if error: | |
| 57 raise error | |
| 58 | |
| 59 def command_passed(self, success_message, patch): | |
| 60 _log.info("command_passed: success_message='%s' patch='%s'" % ( | |
| 61 success_message, patch.id())) | |
| 62 | |
| 63 def command_failed(self, failure_message, script_error, patch): | |
| 64 _log.info("command_failed: failure_message='%s' script_error='%s' patch=
'%s'" % ( | |
| 65 failure_message, script_error, patch.id())) | |
| 66 self._failure_status_id += 1 | |
| 67 return self._failure_status_id | |
| 68 | |
| 69 def refetch_patch(self, patch): | |
| 70 return patch | |
| 71 | |
| 72 def expected_failures(self): | |
| 73 return ExpectedFailures() | |
| 74 | |
| 75 def test_results(self): | |
| 76 return None | |
| 77 | |
| 78 def report_flaky_tests(self, patch, flaky_results, results_archive): | |
| 79 flaky_tests = [result.filename for result in flaky_results] | |
| 80 _log.info("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'"
% (patch.id(), flaky_tests, results_archive.filename)) | |
| 81 | |
| 82 def archive_last_test_results(self, patch): | |
| 83 _log.info("archive_last_test_results: patch='%s'" % patch.id()) | |
| 84 archive = Mock() | |
| 85 archive.filename = "mock-archive-%s.zip" % patch.id() | |
| 86 return archive | |
| 87 | |
| 88 def build_style(self): | |
| 89 return "both" | |
| 90 | |
| 91 def did_pass_testing_ews(self, patch): | |
| 92 return False | |
| 93 | |
| 94 | |
| 95 class FailingTestCommitQueue(MockCommitQueue): | |
| 96 def __init__(self, error_plan, test_failure_plan): | |
| 97 MockCommitQueue.__init__(self, error_plan) | |
| 98 self._test_run_counter = -1 # Special value to indicate tests have neve
r been run. | |
| 99 self._test_failure_plan = test_failure_plan | |
| 100 | |
| 101 def run_command(self, command): | |
| 102 if command[0] == "build-and-test": | |
| 103 self._test_run_counter += 1 | |
| 104 MockCommitQueue.run_command(self, command) | |
| 105 | |
| 106 def _mock_test_result(self, testname): | |
| 107 return test_results.TestResult(testname, [test_failures.FailureTextMisma
tch()]) | |
| 108 | |
| 109 def test_results(self): | |
| 110 # Doesn't make sense to ask for the test_results until the tests have ru
n at least once. | |
| 111 assert(self._test_run_counter >= 0) | |
| 112 failures_for_run = self._test_failure_plan[self._test_run_counter] | |
| 113 results = LayoutTestResults(map(self._mock_test_result, failures_for_run
)) | |
| 114 # This makes the results trustable by ExpectedFailures. | |
| 115 results.set_failure_limit_count(10) | |
| 116 return results | |
| 117 | |
| 118 | |
| 119 # We use GoldenScriptError to make sure that the code under test throws the | |
| 120 # correct (i.e., golden) exception. | |
| 121 class GoldenScriptError(ScriptError): | |
| 122 pass | |
| 123 | |
| 124 | |
| 125 class CommitQueueTaskTest(unittest.TestCase): | |
| 126 def _run_through_task(self, commit_queue, expected_logs, expected_exception=
None, expect_retry=False): | |
| 127 tool = MockTool(log_executive=True) | |
| 128 patch = tool.bugs.fetch_attachment(10000) | |
| 129 task = CommitQueueTask(commit_queue, patch) | |
| 130 success = OutputCapture().assert_outputs(self, task.run, expected_logs=e
xpected_logs, expected_exception=expected_exception) | |
| 131 if not expected_exception: | |
| 132 self.assertEqual(success, not expect_retry) | |
| 133 return task | |
| 134 | |
| 135 def test_success_case(self): | |
| 136 commit_queue = MockCommitQueue([]) | |
| 137 expected_logs = """run_webkit_patch: ['clean'] | |
| 138 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 139 run_webkit_patch: ['update'] | |
| 140 command_passed: success_message='Updated working directory' patch='10000' | |
| 141 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 142 command_passed: success_message='Applied patch' patch='10000' | |
| 143 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 144 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 145 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 146 command_passed: success_message='Built patch' patch='10000' | |
| 147 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 148 command_passed: success_message='Passed tests' patch='10000' | |
| 149 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
| 150 command_passed: success_message='Landed patch' patch='10000' | |
| 151 """ | |
| 152 self._run_through_task(commit_queue, expected_logs) | |
| 153 | |
| 154 def test_fast_success_case(self): | |
| 155 commit_queue = MockCommitQueue([]) | |
| 156 commit_queue.did_pass_testing_ews = lambda patch: True | |
| 157 expected_logs = """run_webkit_patch: ['clean'] | |
| 158 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 159 run_webkit_patch: ['update'] | |
| 160 command_passed: success_message='Updated working directory' patch='10000' | |
| 161 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 162 command_passed: success_message='Applied patch' patch='10000' | |
| 163 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 164 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 165 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 166 command_passed: success_message='Built patch' patch='10000' | |
| 167 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
| 168 command_passed: success_message='Landed patch' patch='10000' | |
| 169 """ | |
| 170 self._run_through_task(commit_queue, expected_logs) | |
| 171 | |
| 172 def test_clean_failure(self): | |
| 173 commit_queue = MockCommitQueue([ | |
| 174 ScriptError("MOCK clean failure"), | |
| 175 ]) | |
| 176 expected_logs = """run_webkit_patch: ['clean'] | |
| 177 command_failed: failure_message='Unable to clean working directory' script_error
='MOCK clean failure' patch='10000' | |
| 178 """ | |
| 179 self._run_through_task(commit_queue, expected_logs, expect_retry=True) | |
| 180 | |
| 181 def test_update_failure(self): | |
| 182 commit_queue = MockCommitQueue([ | |
| 183 None, | |
| 184 ScriptError("MOCK update failure"), | |
| 185 ]) | |
| 186 expected_logs = """run_webkit_patch: ['clean'] | |
| 187 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 188 run_webkit_patch: ['update'] | |
| 189 command_failed: failure_message='Unable to update working directory' script_erro
r='MOCK update failure' patch='10000' | |
| 190 """ | |
| 191 self._run_through_task(commit_queue, expected_logs, expect_retry=True) | |
| 192 | |
| 193 def test_apply_failure(self): | |
| 194 commit_queue = MockCommitQueue([ | |
| 195 None, | |
| 196 None, | |
| 197 GoldenScriptError("MOCK apply failure"), | |
| 198 ]) | |
| 199 expected_logs = """run_webkit_patch: ['clean'] | |
| 200 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 201 run_webkit_patch: ['update'] | |
| 202 command_passed: success_message='Updated working directory' patch='10000' | |
| 203 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 204 command_failed: failure_message='Patch does not apply' script_error='MOCK apply
failure' patch='10000' | |
| 205 """ | |
| 206 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
| 207 | |
| 208 def test_validate_changelog_failure(self): | |
| 209 commit_queue = MockCommitQueue([ | |
| 210 None, | |
| 211 None, | |
| 212 None, | |
| 213 GoldenScriptError("MOCK validate failure"), | |
| 214 ]) | |
| 215 expected_logs = """run_webkit_patch: ['clean'] | |
| 216 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 217 run_webkit_patch: ['update'] | |
| 218 command_passed: success_message='Updated working directory' patch='10000' | |
| 219 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 220 command_passed: success_message='Applied patch' patch='10000' | |
| 221 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 222 command_failed: failure_message='ChangeLog did not pass validation' script_error
='MOCK validate failure' patch='10000' | |
| 223 """ | |
| 224 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
| 225 | |
| 226 def test_build_failure(self): | |
| 227 commit_queue = MockCommitQueue([ | |
| 228 None, | |
| 229 None, | |
| 230 None, | |
| 231 None, | |
| 232 GoldenScriptError("MOCK build failure"), | |
| 233 ]) | |
| 234 expected_logs = """run_webkit_patch: ['clean'] | |
| 235 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 236 run_webkit_patch: ['update'] | |
| 237 command_passed: success_message='Updated working directory' patch='10000' | |
| 238 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 239 command_passed: success_message='Applied patch' patch='10000' | |
| 240 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 241 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 242 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 243 command_failed: failure_message='Patch does not build' script_error='MOCK build
failure' patch='10000' | |
| 244 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both'
] | |
| 245 command_passed: success_message='Able to build without patch' patch='10000' | |
| 246 """ | |
| 247 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
| 248 | |
| 249 def test_red_build_failure(self): | |
| 250 commit_queue = MockCommitQueue([ | |
| 251 None, | |
| 252 None, | |
| 253 None, | |
| 254 None, | |
| 255 ScriptError("MOCK build failure"), | |
| 256 ScriptError("MOCK clean build failure"), | |
| 257 ]) | |
| 258 expected_logs = """run_webkit_patch: ['clean'] | |
| 259 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 260 run_webkit_patch: ['update'] | |
| 261 command_passed: success_message='Updated working directory' patch='10000' | |
| 262 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 263 command_passed: success_message='Applied patch' patch='10000' | |
| 264 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 265 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 266 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 267 command_failed: failure_message='Patch does not build' script_error='MOCK build
failure' patch='10000' | |
| 268 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both'
] | |
| 269 command_failed: failure_message='Unable to build without patch' script_error='MO
CK clean build failure' patch='10000' | |
| 270 """ | |
| 271 self._run_through_task(commit_queue, expected_logs, expect_retry=True) | |
| 272 | |
| 273 def test_flaky_test_failure(self): | |
| 274 commit_queue = MockCommitQueue([ | |
| 275 None, | |
| 276 None, | |
| 277 None, | |
| 278 None, | |
| 279 None, | |
| 280 ScriptError("MOCK tests failure"), | |
| 281 ]) | |
| 282 # CommitQueueTask will only report flaky tests if we successfully parsed | |
| 283 # results.json and returned a LayoutTestResults object, so we fake one. | |
| 284 commit_queue.test_results = lambda: LayoutTestResults([]) | |
| 285 expected_logs = """run_webkit_patch: ['clean'] | |
| 286 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 287 run_webkit_patch: ['update'] | |
| 288 command_passed: success_message='Updated working directory' patch='10000' | |
| 289 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 290 command_passed: success_message='Applied patch' patch='10000' | |
| 291 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 292 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 293 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 294 command_passed: success_message='Built patch' patch='10000' | |
| 295 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 296 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
ests failure' patch='10000' | |
| 297 archive_last_test_results: patch='10000' | |
| 298 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 299 command_passed: success_message='Passed tests' patch='10000' | |
| 300 report_flaky_tests: patch='10000' flaky_tests='[]' archive='mock-archive-10000.z
ip' | |
| 301 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
| 302 command_passed: success_message='Landed patch' patch='10000' | |
| 303 """ | |
| 304 self._run_through_task(commit_queue, expected_logs) | |
| 305 | |
| 306 def test_failed_archive(self): | |
| 307 commit_queue = MockCommitQueue([ | |
| 308 None, | |
| 309 None, | |
| 310 None, | |
| 311 None, | |
| 312 None, | |
| 313 ScriptError("MOCK tests failure"), | |
| 314 ]) | |
| 315 commit_queue.test_results = lambda: LayoutTestResults([]) | |
| 316 # It's possible delegate to fail to archive layout tests, don't try to r
eport | |
| 317 # flaky tests when that happens. | |
| 318 commit_queue.archive_last_test_results = lambda patch: None | |
| 319 expected_logs = """run_webkit_patch: ['clean'] | |
| 320 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 321 run_webkit_patch: ['update'] | |
| 322 command_passed: success_message='Updated working directory' patch='10000' | |
| 323 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 324 command_passed: success_message='Applied patch' patch='10000' | |
| 325 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 326 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 327 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 328 command_passed: success_message='Built patch' patch='10000' | |
| 329 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 330 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
ests failure' patch='10000' | |
| 331 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 332 command_passed: success_message='Passed tests' patch='10000' | |
| 333 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
| 334 command_passed: success_message='Landed patch' patch='10000' | |
| 335 """ | |
| 336 self._run_through_task(commit_queue, expected_logs) | |
| 337 | |
| 338 def test_double_flaky_test_failure(self): | |
| 339 commit_queue = FailingTestCommitQueue([ | |
| 340 None, | |
| 341 None, | |
| 342 None, | |
| 343 None, | |
| 344 None, | |
| 345 ScriptError("MOCK test failure"), | |
| 346 ScriptError("MOCK test failure again"), | |
| 347 ], [ | |
| 348 "foo.html", | |
| 349 "bar.html", | |
| 350 "foo.html", | |
| 351 ]) | |
| 352 # The (subtle) point of this test is that report_flaky_tests does not ap
pear | |
| 353 # in the expected_logs for this run. | |
| 354 # Note also that there is no attempt to run the tests w/o the patch. | |
| 355 expected_logs = """run_webkit_patch: ['clean'] | |
| 356 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 357 run_webkit_patch: ['update'] | |
| 358 command_passed: success_message='Updated working directory' patch='10000' | |
| 359 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 360 command_passed: success_message='Applied patch' patch='10000' | |
| 361 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 362 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 363 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 364 command_passed: success_message='Built patch' patch='10000' | |
| 365 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 366 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
| 367 archive_last_test_results: patch='10000' | |
| 368 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 369 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
| 370 """ | |
| 371 tool = MockTool(log_executive=True) | |
| 372 patch = tool.bugs.fetch_attachment(10000) | |
| 373 task = CommitQueueTask(commit_queue, patch) | |
| 374 success = OutputCapture().assert_outputs(self, task.run, expected_logs=e
xpected_logs) | |
| 375 self.assertFalse(success) | |
| 376 | |
| 377 def test_test_failure(self): | |
| 378 commit_queue = MockCommitQueue([ | |
| 379 None, | |
| 380 None, | |
| 381 None, | |
| 382 None, | |
| 383 None, | |
| 384 GoldenScriptError("MOCK test failure"), | |
| 385 ScriptError("MOCK test failure again"), | |
| 386 ]) | |
| 387 expected_logs = """run_webkit_patch: ['clean'] | |
| 388 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 389 run_webkit_patch: ['update'] | |
| 390 command_passed: success_message='Updated working directory' patch='10000' | |
| 391 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 392 command_passed: success_message='Applied patch' patch='10000' | |
| 393 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 394 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 395 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 396 command_passed: success_message='Built patch' patch='10000' | |
| 397 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 398 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
| 399 archive_last_test_results: patch='10000' | |
| 400 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 401 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
| 402 archive_last_test_results: patch='10000' | |
| 403 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build',
'--test', '--non-interactive'] | |
| 404 command_passed: success_message='Able to pass tests without patch' patch='10000' | |
| 405 """ | |
| 406 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
| 407 | |
| 408 def test_red_test_failure(self): | |
| 409 commit_queue = FailingTestCommitQueue([ | |
| 410 None, | |
| 411 None, | |
| 412 None, | |
| 413 None, | |
| 414 None, | |
| 415 ScriptError("MOCK test failure"), | |
| 416 ScriptError("MOCK test failure again"), | |
| 417 ScriptError("MOCK clean test failure"), | |
| 418 ], [ | |
| 419 "foo.html", | |
| 420 "foo.html", | |
| 421 "foo.html", | |
| 422 ]) | |
| 423 | |
| 424 # Tests always fail, and always return the same results, but we | |
| 425 # should still be able to land in this case! | |
| 426 expected_logs = """run_webkit_patch: ['clean'] | |
| 427 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 428 run_webkit_patch: ['update'] | |
| 429 command_passed: success_message='Updated working directory' patch='10000' | |
| 430 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 431 command_passed: success_message='Applied patch' patch='10000' | |
| 432 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 433 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 434 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 435 command_passed: success_message='Built patch' patch='10000' | |
| 436 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 437 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
| 438 archive_last_test_results: patch='10000' | |
| 439 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 440 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
| 441 archive_last_test_results: patch='10000' | |
| 442 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build',
'--test', '--non-interactive'] | |
| 443 command_failed: failure_message='Unable to pass tests without patch (tree is red
?)' script_error='MOCK clean test failure' patch='10000' | |
| 444 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
| 445 command_passed: success_message='Landed patch' patch='10000' | |
| 446 """ | |
| 447 self._run_through_task(commit_queue, expected_logs) | |
| 448 | |
| 449 def test_very_red_tree_retry(self): | |
| 450 lots_of_failing_tests = map(lambda num: "test-%s.html" % num, range(0, 1
00)) | |
| 451 commit_queue = FailingTestCommitQueue([ | |
| 452 None, | |
| 453 None, | |
| 454 None, | |
| 455 None, | |
| 456 None, | |
| 457 ScriptError("MOCK test failure"), | |
| 458 ScriptError("MOCK test failure again"), | |
| 459 ScriptError("MOCK clean test failure"), | |
| 460 ], [ | |
| 461 lots_of_failing_tests, | |
| 462 lots_of_failing_tests, | |
| 463 lots_of_failing_tests, | |
| 464 ]) | |
| 465 | |
| 466 # Tests always fail, and return so many failures that we do not | |
| 467 # trust the results (see ExpectedFailures._can_trust_results) so we | |
| 468 # just give up and retry the patch. | |
| 469 expected_logs = """run_webkit_patch: ['clean'] | |
| 470 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 471 run_webkit_patch: ['update'] | |
| 472 command_passed: success_message='Updated working directory' patch='10000' | |
| 473 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 474 command_passed: success_message='Applied patch' patch='10000' | |
| 475 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 476 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 477 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 478 command_passed: success_message='Built patch' patch='10000' | |
| 479 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 480 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
| 481 archive_last_test_results: patch='10000' | |
| 482 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 483 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
| 484 archive_last_test_results: patch='10000' | |
| 485 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build',
'--test', '--non-interactive'] | |
| 486 command_failed: failure_message='Unable to pass tests without patch (tree is red
?)' script_error='MOCK clean test failure' patch='10000' | |
| 487 """ | |
| 488 self._run_through_task(commit_queue, expected_logs, expect_retry=True) | |
| 489 | |
| 490 def test_red_tree_patch_rejection(self): | |
| 491 commit_queue = FailingTestCommitQueue([ | |
| 492 None, | |
| 493 None, | |
| 494 None, | |
| 495 None, | |
| 496 None, | |
| 497 GoldenScriptError("MOCK test failure"), | |
| 498 ScriptError("MOCK test failure again"), | |
| 499 ScriptError("MOCK clean test failure"), | |
| 500 ], [ | |
| 501 ["foo.html", "bar.html"], | |
| 502 ["foo.html", "bar.html"], | |
| 503 ["foo.html"], | |
| 504 ]) | |
| 505 | |
| 506 # Tests always fail, but the clean tree only fails one test | |
| 507 # while the patch fails two. So we should reject the patch! | |
| 508 expected_logs = """run_webkit_patch: ['clean'] | |
| 509 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 510 run_webkit_patch: ['update'] | |
| 511 command_passed: success_message='Updated working directory' patch='10000' | |
| 512 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 513 command_passed: success_message='Applied patch' patch='10000' | |
| 514 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 515 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 516 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 517 command_passed: success_message='Built patch' patch='10000' | |
| 518 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 519 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure' patch='10000' | |
| 520 archive_last_test_results: patch='10000' | |
| 521 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 522 command_failed: failure_message='Patch does not pass tests' script_error='MOCK t
est failure again' patch='10000' | |
| 523 archive_last_test_results: patch='10000' | |
| 524 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build',
'--test', '--non-interactive'] | |
| 525 command_failed: failure_message='Unable to pass tests without patch (tree is red
?)' script_error='MOCK clean test failure' patch='10000' | |
| 526 """ | |
| 527 task = self._run_through_task(commit_queue, expected_logs, GoldenScriptE
rror) | |
| 528 self.assertEqual(task.results_from_patch_test_run(task._patch).failing_t
ests(), ["foo.html", "bar.html"]) | |
| 529 # failure_status_id should be of the test with patch (1), not the test w
ithout patch (2). | |
| 530 self.assertEqual(task.failure_status_id, 1) | |
| 531 | |
| 532 def test_land_failure(self): | |
| 533 commit_queue = MockCommitQueue([ | |
| 534 None, | |
| 535 None, | |
| 536 None, | |
| 537 None, | |
| 538 None, | |
| 539 None, | |
| 540 GoldenScriptError("MOCK land failure"), | |
| 541 ]) | |
| 542 expected_logs = """run_webkit_patch: ['clean'] | |
| 543 command_passed: success_message='Cleaned working directory' patch='10000' | |
| 544 run_webkit_patch: ['update'] | |
| 545 command_passed: success_message='Updated working directory' patch='10000' | |
| 546 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000
] | |
| 547 command_passed: success_message='Applied patch' patch='10000' | |
| 548 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000] | |
| 549 command_passed: success_message='ChangeLog validated' patch='10000' | |
| 550 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] | |
| 551 command_passed: success_message='Built patch' patch='10000' | |
| 552 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--n
on-interactive'] | |
| 553 command_passed: success_message='Passed tests' patch='10000' | |
| 554 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--p
arent-command=commit-queue', 10000] | |
| 555 command_failed: failure_message='Unable to land patch' script_error='MOCK land f
ailure' patch='10000' | |
| 556 """ | |
| 557 # FIXME: This should really be expect_retry=True for a better user exper
iance. | |
| 558 self._run_through_task(commit_queue, expected_logs, GoldenScriptError) | |
| 559 | |
| 560 def _expect_validate(self, patch, is_valid): | |
| 561 class MockDelegate(object): | |
| 562 def refetch_patch(self, patch): | |
| 563 return patch | |
| 564 | |
| 565 def expected_failures(self): | |
| 566 return ExpectedFailures() | |
| 567 | |
| 568 task = CommitQueueTask(MockDelegate(), patch) | |
| 569 self.assertEqual(task.validate(), is_valid) | |
| 570 | |
| 571 def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, co
mmitter="fake"): | |
| 572 bug = bugzilla.Bug(bug_dict, None) | |
| 573 patch = bugzilla.Attachment(attachment_dict, bug) | |
| 574 patch._committer = committer | |
| 575 return patch | |
| 576 | |
| 577 def test_validate(self): | |
| 578 self._expect_validate(self._mock_patch(), True) | |
| 579 self._expect_validate(self._mock_patch({'is_obsolete': True}), False) | |
| 580 self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}
), False) | |
| 581 self._expect_validate(self._mock_patch(committer=None), False) | |
| 582 self._expect_validate(self._mock_patch({'review': '-'}), False) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 15416008:
Remove a bunch of dead code from webkitpy (Closed)
Base URL: svn://svn.chromium.org/blink/trunk