| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # | 2 # |
| 3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 """Runs all the native unit tests. | 7 """Runs all the native unit tests. |
| 8 | 8 |
| 9 1. Copy over test binary to /data/local on device. | 9 1. Copy over test binary to /data/local on device. |
| 10 2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak) | 10 2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak) |
| (...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 75 'ipc_tests', | 75 'ipc_tests', |
| 76 'media_unittests', | 76 'media_unittests', |
| 77 'net_unittests', | 77 'net_unittests', |
| 78 'sql_unittests', | 78 'sql_unittests', |
| 79 'sync_unit_tests', | 79 'sync_unit_tests', |
| 80 'ui_unittests', | 80 'ui_unittests', |
| 81 'unit_tests', | 81 'unit_tests', |
| 82 ] | 82 ] |
| 83 | 83 |
| 84 | 84 |
| 85 def FullyQualifiedTestSuites(exe, option_test_suite): | 85 def TestSuiteDir(build_type): |
| 86 """Return the base directory of test suites.""" |
| 87 return os.path.abspath(os.path.join(constants.CHROME_DIR, 'out', build_type)) |
| 88 |
| 89 def FullyQualifiedTestSuites(exe, option_test_suite, build_type): |
| 86 """Return a fully qualified list | 90 """Return a fully qualified list |
| 87 | 91 |
| 88 Args: | 92 Args: |
| 89 exe: if True, use the executable-based test runner. | 93 exe: if True, use the executable-based test runner. |
| 90 option_test_suite: the test_suite specified as an option. | 94 option_test_suite: the test_suite specified as an option. |
| 95 build_type: 'Release' or 'Debug'. |
| 91 """ | 96 """ |
| 92 # Assume the test suites are in out/Release. | 97 test_suite_dir = TestSuiteDir(build_type) |
| 93 test_suite_dir = os.path.abspath(os.path.join(constants.CHROME_DIR, | |
| 94 'out', 'Release')) | |
| 95 if option_test_suite: | 98 if option_test_suite: |
| 96 all_test_suites = [option_test_suite] | 99 all_test_suites = [option_test_suite] |
| 97 else: | 100 else: |
| 98 all_test_suites = _TEST_SUITES | 101 all_test_suites = _TEST_SUITES |
| 99 | 102 |
| 100 if exe: | 103 if exe: |
| 101 qualified_test_suites = [os.path.join(test_suite_dir, t) | 104 qualified_test_suites = [os.path.join(test_suite_dir, t) |
| 102 for t in all_test_suites] | 105 for t in all_test_suites] |
| 103 else: | 106 else: |
| 104 # out/Release/$SUITE_apk/$SUITE-debug.apk | 107 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk |
| 105 qualified_test_suites = [os.path.join(test_suite_dir, | 108 qualified_test_suites = [os.path.join(test_suite_dir, |
| 106 t + '_apk', | 109 t + '_apk', |
| 107 t + '-debug.apk') | 110 t + '-debug.apk') |
| 108 for t in all_test_suites] | 111 for t in all_test_suites] |
| 109 for t, q in zip(all_test_suites, qualified_test_suites): | 112 for t, q in zip(all_test_suites, qualified_test_suites): |
| 110 if not os.path.exists(q): | 113 if not os.path.exists(q): |
| 111 logging.critical('Test suite %s not found in %s.\n' | 114 logging.critical('Test suite %s not found in %s.\n' |
| 112 'Supported test suites:\n %s\n' | 115 'Supported test suites:\n %s\n' |
| 113 'Ensure it has been built.\n', | 116 'Ensure it has been built.\n', |
| 114 t, q, _TEST_SUITES) | 117 t, q, _TEST_SUITES) |
| (...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 188 buildbot_report.PrintError() | 191 buildbot_report.PrintError() |
| 189 else: | 192 else: |
| 190 print 'Step success!' # No annotation needed | 193 print 'Step success!' # No annotation needed |
| 191 | 194 |
| 192 | 195 |
| 193 class TestSharder(BaseTestSharder): | 196 class TestSharder(BaseTestSharder): |
| 194 """Responsible for sharding the tests on the connected devices.""" | 197 """Responsible for sharding the tests on the connected devices.""" |
| 195 | 198 |
| 196 def __init__(self, attached_devices, test_suite, gtest_filter, | 199 def __init__(self, attached_devices, test_suite, gtest_filter, |
| 197 test_arguments, timeout, rebaseline, performance_test, | 200 test_arguments, timeout, rebaseline, performance_test, |
| 198 cleanup_test_files, tool, log_dump_name, fast_and_loose): | 201 cleanup_test_files, tool, log_dump_name, fast_and_loose, |
| 202 build_type): |
| 199 BaseTestSharder.__init__(self, attached_devices) | 203 BaseTestSharder.__init__(self, attached_devices) |
| 200 self.test_suite = test_suite | 204 self.test_suite = test_suite |
| 201 self.test_suite_basename = os.path.basename(test_suite) | 205 self.test_suite_basename = os.path.basename(test_suite) |
| 202 self.gtest_filter = gtest_filter or '' | 206 self.gtest_filter = gtest_filter or '' |
| 203 self.test_arguments = test_arguments | 207 self.test_arguments = test_arguments |
| 204 self.timeout = timeout | 208 self.timeout = timeout |
| 205 self.rebaseline = rebaseline | 209 self.rebaseline = rebaseline |
| 206 self.performance_test = performance_test | 210 self.performance_test = performance_test |
| 207 self.cleanup_test_files = cleanup_test_files | 211 self.cleanup_test_files = cleanup_test_files |
| 208 self.tool = tool | 212 self.tool = tool |
| 209 self.log_dump_name = log_dump_name | 213 self.log_dump_name = log_dump_name |
| 210 self.fast_and_loose = fast_and_loose | 214 self.fast_and_loose = fast_and_loose |
| 215 self.build_type = build_type |
| 211 test = SingleTestRunner(self.attached_devices[0], test_suite, gtest_filter, | 216 test = SingleTestRunner(self.attached_devices[0], test_suite, gtest_filter, |
| 212 test_arguments, timeout, rebaseline, | 217 test_arguments, timeout, rebaseline, |
| 213 performance_test, cleanup_test_files, tool, 0, | 218 performance_test, cleanup_test_files, tool, 0, |
| 214 not not self.log_dump_name, fast_and_loose) | 219 not not self.log_dump_name, fast_and_loose, |
| 220 build_type) |
| 215 self.tests = [] | 221 self.tests = [] |
| 216 if not self.gtest_filter: | 222 if not self.gtest_filter: |
| 217 # No filter has been specified, let's add all tests then. | 223 # No filter has been specified, let's add all tests then. |
| 218 # The executable/apk needs to be copied before we can call GetAllTests. | 224 # The executable/apk needs to be copied before we can call GetAllTests. |
| 219 test.test_package.StripAndCopyExecutable() | 225 test.test_package.StripAndCopyExecutable() |
| 220 all_tests = test.test_package.GetAllTests() | 226 all_tests = test.test_package.GetAllTests() |
| 221 if not rebaseline: | 227 if not rebaseline: |
| 222 disabled_list = test.GetDisabledTests() | 228 disabled_list = test.GetDisabledTests() |
| 223 # Only includes tests that do not have any match in the disabled list. | 229 # Only includes tests that do not have any match in the disabled list. |
| 224 all_tests = filter(lambda t: | 230 all_tests = filter(lambda t: |
| (...skipping 13 matching lines...) Expand all Loading... |
| 238 A SingleTestRunner object. | 244 A SingleTestRunner object. |
| 239 """ | 245 """ |
| 240 device_num = len(self.attached_devices) | 246 device_num = len(self.attached_devices) |
| 241 shard_size = (len(self.tests) + device_num - 1) / device_num | 247 shard_size = (len(self.tests) + device_num - 1) / device_num |
| 242 shard_test_list = self.tests[index * shard_size : (index + 1) * shard_size] | 248 shard_test_list = self.tests[index * shard_size : (index + 1) * shard_size] |
| 243 test_filter = ':'.join(shard_test_list) + self.gtest_filter | 249 test_filter = ':'.join(shard_test_list) + self.gtest_filter |
| 244 return SingleTestRunner(device, self.test_suite, | 250 return SingleTestRunner(device, self.test_suite, |
| 245 test_filter, self.test_arguments, self.timeout, | 251 test_filter, self.test_arguments, self.timeout, |
| 246 self.rebaseline, self.performance_test, | 252 self.rebaseline, self.performance_test, |
| 247 self.cleanup_test_files, self.tool, index, | 253 self.cleanup_test_files, self.tool, index, |
| 248 not not self.log_dump_name, self.fast_and_loose) | 254 not not self.log_dump_name, self.fast_and_loose, |
| 255 self.build_type) |
| 249 | 256 |
| 250 def OnTestsCompleted(self, test_runners, test_results): | 257 def OnTestsCompleted(self, test_runners, test_results): |
| 251 """Notifies that we completed the tests.""" | 258 """Notifies that we completed the tests.""" |
| 252 test_results.LogFull('Unit test', os.path.basename(self.test_suite)) | 259 test_results.LogFull('Unit test', os.path.basename(self.test_suite)) |
| 253 PrintAnnotationForTestResults(test_results) | 260 PrintAnnotationForTestResults(test_results) |
| 254 if test_results.failed and self.rebaseline: | 261 if test_results.failed and self.rebaseline: |
| 255 test_runners[0].UpdateFilter(test_results.failed) | 262 test_runners[0].UpdateFilter(test_results.failed) |
| 256 if self.log_dump_name: | 263 if self.log_dump_name: |
| 257 # Zip all debug info outputs into a file named by log_dump_name. | 264 # Zip all debug info outputs into a file named by log_dump_name. |
| 258 debug_info.GTestDebugInfo.ZipAndCleanResults( | 265 debug_info.GTestDebugInfo.ZipAndCleanResults( |
| 259 os.path.join(constants.CHROME_DIR, 'out', 'Release', | 266 os.path.join(TestSuiteDir(self.build_type), 'debug_info_dumps'), |
| 260 'debug_info_dumps'), | |
| 261 self.log_dump_name) | 267 self.log_dump_name) |
| 262 | 268 |
| 263 | 269 |
| 264 def _RunATestSuite(options): | 270 def _RunATestSuite(options): |
| 265 """Run a single test suite. | 271 """Run a single test suite. |
| 266 | 272 |
| 267 Helper for Dispatch() to allow stop/restart of the emulator across | 273 Helper for Dispatch() to allow stop/restart of the emulator across |
| 268 test bundles. If using the emulator, we start it on entry and stop | 274 test bundles. If using the emulator, we start it on entry and stop |
| 269 it on exit. | 275 it on exit. |
| 270 | 276 |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 310 raise Exception('Failed to reset test server port.') | 316 raise Exception('Failed to reset test server port.') |
| 311 | 317 |
| 312 if options.performance_test or options.gtest_filter: | 318 if options.performance_test or options.gtest_filter: |
| 313 # These configuration can't be split in multiple devices. | 319 # These configuration can't be split in multiple devices. |
| 314 attached_devices = [attached_devices[0]] | 320 attached_devices = [attached_devices[0]] |
| 315 sharder = TestSharder(attached_devices, options.test_suite, | 321 sharder = TestSharder(attached_devices, options.test_suite, |
| 316 options.gtest_filter, options.test_arguments, | 322 options.gtest_filter, options.test_arguments, |
| 317 options.timeout, options.rebaseline, | 323 options.timeout, options.rebaseline, |
| 318 options.performance_test, | 324 options.performance_test, |
| 319 options.cleanup_test_files, options.tool, | 325 options.cleanup_test_files, options.tool, |
| 320 options.log_dump, options.fast_and_loose) | 326 options.log_dump, options.fast_and_loose, |
| 327 options.build_type) |
| 321 test_results = sharder.RunShardedTests() | 328 test_results = sharder.RunShardedTests() |
| 322 | 329 |
| 323 for buildbot_emulator in buildbot_emulators: | 330 for buildbot_emulator in buildbot_emulators: |
| 324 buildbot_emulator.Shutdown() | 331 buildbot_emulator.Shutdown() |
| 325 | 332 |
| 326 # Another chance if we timed out? At this point It is safe(r) to | 333 # Another chance if we timed out? At this point It is safe(r) to |
| 327 # run fast and loose since we just uploaded all the test data and | 334 # run fast and loose since we just uploaded all the test data and |
| 328 # binary. | 335 # binary. |
| 329 if test_results.timed_out and options.repeat: | 336 if test_results.timed_out and options.repeat: |
| 330 logging.critical('Timed out; repeating in fast_and_loose mode.') | 337 logging.critical('Timed out; repeating in fast_and_loose mode.') |
| (...skipping 17 matching lines...) Expand all Loading... |
| 348 0 if successful, number of failing tests otherwise. | 355 0 if successful, number of failing tests otherwise. |
| 349 """ | 356 """ |
| 350 if options.test_suite == 'help': | 357 if options.test_suite == 'help': |
| 351 ListTestSuites() | 358 ListTestSuites() |
| 352 return 0 | 359 return 0 |
| 353 | 360 |
| 354 if options.use_xvfb: | 361 if options.use_xvfb: |
| 355 xvfb = Xvfb() | 362 xvfb = Xvfb() |
| 356 xvfb.Start() | 363 xvfb.Start() |
| 357 | 364 |
| 358 all_test_suites = FullyQualifiedTestSuites(options.exe, options.test_suite) | 365 all_test_suites = FullyQualifiedTestSuites(options.exe, options.test_suite, |
| 366 options.build_type) |
| 359 failures = 0 | 367 failures = 0 |
| 360 for suite in all_test_suites: | 368 for suite in all_test_suites: |
| 361 options.test_suite = suite | 369 options.test_suite = suite |
| 362 failures += _RunATestSuite(options) | 370 failures += _RunATestSuite(options) |
| 363 | 371 |
| 364 if options.use_xvfb: | 372 if options.use_xvfb: |
| 365 xvfb.Stop() | 373 xvfb.Stop() |
| 366 return failures | 374 return failures |
| 367 | 375 |
| 368 | 376 |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 415 'Don\'t use on bots by default!') | 423 'Don\'t use on bots by default!') |
| 416 option_parser.add_option('--repeat', dest='repeat', type='int', | 424 option_parser.add_option('--repeat', dest='repeat', type='int', |
| 417 default=2, | 425 default=2, |
| 418 help='Repeat count on test timeout') | 426 help='Repeat count on test timeout') |
| 419 option_parser.add_option('--exit_code', action='store_true', | 427 option_parser.add_option('--exit_code', action='store_true', |
| 420 help='If set, the exit code will be total number ' | 428 help='If set, the exit code will be total number ' |
| 421 'of failures.') | 429 'of failures.') |
| 422 option_parser.add_option('--exe', action='store_true', | 430 option_parser.add_option('--exe', action='store_true', |
| 423 help='If set, use the exe test runner instead of ' | 431 help='If set, use the exe test runner instead of ' |
| 424 'the APK.') | 432 'the APK.') |
| 433 |
| 425 options, args = option_parser.parse_args(argv) | 434 options, args = option_parser.parse_args(argv) |
| 426 if len(args) > 1: | 435 if len(args) > 1: |
| 427 print 'Unknown argument:', args[1:] | 436 print 'Unknown argument:', args[1:] |
| 428 option_parser.print_usage() | 437 option_parser.print_usage() |
| 429 sys.exit(1) | 438 sys.exit(1) |
| 430 run_tests_helper.SetLogLevel(options.verbose_count) | 439 run_tests_helper.SetLogLevel(options.verbose_count) |
| 431 emulator.DeleteAllTempAVDs() | 440 emulator.DeleteAllTempAVDs() |
| 432 failed_tests_count = Dispatch(options) | 441 failed_tests_count = Dispatch(options) |
| 433 | 442 |
| 434 # Failures of individual test suites are communicated by printing a | 443 # Failures of individual test suites are communicated by printing a |
| 435 # STEP_FAILURE message. | 444 # STEP_FAILURE message. |
| 436 # Returning a success exit status also prevents the buildbot from incorrectly | 445 # Returning a success exit status also prevents the buildbot from incorrectly |
| 437 # marking the last suite as failed if there were failures in other suites in | 446 # marking the last suite as failed if there were failures in other suites in |
| 438 # the batch (this happens because the exit status is a sum of all failures | 447 # the batch (this happens because the exit status is a sum of all failures |
| 439 # from all suites, but the buildbot associates the exit status only with the | 448 # from all suites, but the buildbot associates the exit status only with the |
| 440 # most recent step). | 449 # most recent step). |
| 441 if options.exit_code: | 450 if options.exit_code: |
| 442 return failed_tests_count | 451 return failed_tests_count |
| 443 return 0 | 452 return 0 |
| 444 | 453 |
| 445 | 454 |
| 446 if __name__ == '__main__': | 455 if __name__ == '__main__': |
| 447 sys.exit(main(sys.argv)) | 456 sys.exit(main(sys.argv)) |
| OLD | NEW |