OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all the native unit tests. | 7 """Runs all the native unit tests. |
8 | 8 |
9 1. Copy over test binary to /data/local on device. | 9 1. Copy over test binary to /data/local on device. |
10 2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak) | 10 2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak) |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
75 'ipc_tests', | 75 'ipc_tests', |
76 'media_unittests', | 76 'media_unittests', |
77 'net_unittests', | 77 'net_unittests', |
78 'sql_unittests', | 78 'sql_unittests', |
79 'sync_unit_tests', | 79 'sync_unit_tests', |
80 'ui_unittests', | 80 'ui_unittests', |
81 'unit_tests', | 81 'unit_tests', |
82 ] | 82 ] |
83 | 83 |
84 | 84 |
85 def FullyQualifiedTestSuites(exe, option_test_suite): | 85 def TestSuiteDir(build_type): |
| 86 """Return the base directory of test suites.""" |
| 87 return os.path.abspath(os.path.join(constants.CHROME_DIR, 'out', build_type)) |
| 88 |
| 89 def FullyQualifiedTestSuites(exe, option_test_suite, build_type): |
86 """Return a fully qualified list | 90 """Return a fully qualified list |
87 | 91 |
88 Args: | 92 Args: |
89 exe: if True, use the executable-based test runner. | 93 exe: if True, use the executable-based test runner. |
90 option_test_suite: the test_suite specified as an option. | 94 option_test_suite: the test_suite specified as an option. |
| 95 build_type: 'Release' or 'Debug'. |
91 """ | 96 """ |
92 # Assume the test suites are in out/Release. | 97 test_suite_dir = TestSuiteDir(build_type) |
93 test_suite_dir = os.path.abspath(os.path.join(constants.CHROME_DIR, | |
94 'out', 'Release')) | |
95 if option_test_suite: | 98 if option_test_suite: |
96 all_test_suites = [option_test_suite] | 99 all_test_suites = [option_test_suite] |
97 else: | 100 else: |
98 all_test_suites = _TEST_SUITES | 101 all_test_suites = _TEST_SUITES |
99 | 102 |
100 if exe: | 103 if exe: |
101 qualified_test_suites = [os.path.join(test_suite_dir, t) | 104 qualified_test_suites = [os.path.join(test_suite_dir, t) |
102 for t in all_test_suites] | 105 for t in all_test_suites] |
103 else: | 106 else: |
104 # out/Release/$SUITE_apk/$SUITE-debug.apk | 107 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk |
105 qualified_test_suites = [os.path.join(test_suite_dir, | 108 qualified_test_suites = [os.path.join(test_suite_dir, |
106 t + '_apk', | 109 t + '_apk', |
107 t + '-debug.apk') | 110 t + '-debug.apk') |
108 for t in all_test_suites] | 111 for t in all_test_suites] |
109 for t, q in zip(all_test_suites, qualified_test_suites): | 112 for t, q in zip(all_test_suites, qualified_test_suites): |
110 if not os.path.exists(q): | 113 if not os.path.exists(q): |
111 logging.critical('Test suite %s not found in %s.\n' | 114 logging.critical('Test suite %s not found in %s.\n' |
112 'Supported test suites:\n %s\n' | 115 'Supported test suites:\n %s\n' |
113 'Ensure it has been built.\n', | 116 'Ensure it has been built.\n', |
114 t, q, _TEST_SUITES) | 117 t, q, _TEST_SUITES) |
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
188 buildbot_report.PrintError() | 191 buildbot_report.PrintError() |
189 else: | 192 else: |
190 print 'Step success!' # No annotation needed | 193 print 'Step success!' # No annotation needed |
191 | 194 |
192 | 195 |
193 class TestSharder(BaseTestSharder): | 196 class TestSharder(BaseTestSharder): |
194 """Responsible for sharding the tests on the connected devices.""" | 197 """Responsible for sharding the tests on the connected devices.""" |
195 | 198 |
196 def __init__(self, attached_devices, test_suite, gtest_filter, | 199 def __init__(self, attached_devices, test_suite, gtest_filter, |
197 test_arguments, timeout, rebaseline, performance_test, | 200 test_arguments, timeout, rebaseline, performance_test, |
198 cleanup_test_files, tool, log_dump_name, fast_and_loose): | 201 cleanup_test_files, tool, log_dump_name, fast_and_loose, |
| 202 build_type): |
199 BaseTestSharder.__init__(self, attached_devices) | 203 BaseTestSharder.__init__(self, attached_devices) |
200 self.test_suite = test_suite | 204 self.test_suite = test_suite |
201 self.test_suite_basename = os.path.basename(test_suite) | 205 self.test_suite_basename = os.path.basename(test_suite) |
202 self.gtest_filter = gtest_filter or '' | 206 self.gtest_filter = gtest_filter or '' |
203 self.test_arguments = test_arguments | 207 self.test_arguments = test_arguments |
204 self.timeout = timeout | 208 self.timeout = timeout |
205 self.rebaseline = rebaseline | 209 self.rebaseline = rebaseline |
206 self.performance_test = performance_test | 210 self.performance_test = performance_test |
207 self.cleanup_test_files = cleanup_test_files | 211 self.cleanup_test_files = cleanup_test_files |
208 self.tool = tool | 212 self.tool = tool |
209 self.log_dump_name = log_dump_name | 213 self.log_dump_name = log_dump_name |
210 self.fast_and_loose = fast_and_loose | 214 self.fast_and_loose = fast_and_loose |
| 215 self.build_type = build_type |
211 test = SingleTestRunner(self.attached_devices[0], test_suite, gtest_filter, | 216 test = SingleTestRunner(self.attached_devices[0], test_suite, gtest_filter, |
212 test_arguments, timeout, rebaseline, | 217 test_arguments, timeout, rebaseline, |
213 performance_test, cleanup_test_files, tool, 0, | 218 performance_test, cleanup_test_files, tool, 0, |
214 not not self.log_dump_name, fast_and_loose) | 219 not not self.log_dump_name, fast_and_loose, |
| 220 build_type) |
215 self.tests = [] | 221 self.tests = [] |
216 if not self.gtest_filter: | 222 if not self.gtest_filter: |
217 # No filter has been specified, let's add all tests then. | 223 # No filter has been specified, let's add all tests then. |
218 # The executable/apk needs to be copied before we can call GetAllTests. | 224 # The executable/apk needs to be copied before we can call GetAllTests. |
219 test.test_package.StripAndCopyExecutable() | 225 test.test_package.StripAndCopyExecutable() |
220 all_tests = test.test_package.GetAllTests() | 226 all_tests = test.test_package.GetAllTests() |
221 if not rebaseline: | 227 if not rebaseline: |
222 disabled_list = test.GetDisabledTests() | 228 disabled_list = test.GetDisabledTests() |
223 # Only includes tests that do not have any match in the disabled list. | 229 # Only includes tests that do not have any match in the disabled list. |
224 all_tests = filter(lambda t: | 230 all_tests = filter(lambda t: |
(...skipping 13 matching lines...) Expand all Loading... |
238 A SingleTestRunner object. | 244 A SingleTestRunner object. |
239 """ | 245 """ |
240 device_num = len(self.attached_devices) | 246 device_num = len(self.attached_devices) |
241 shard_size = (len(self.tests) + device_num - 1) / device_num | 247 shard_size = (len(self.tests) + device_num - 1) / device_num |
242 shard_test_list = self.tests[index * shard_size : (index + 1) * shard_size] | 248 shard_test_list = self.tests[index * shard_size : (index + 1) * shard_size] |
243 test_filter = ':'.join(shard_test_list) + self.gtest_filter | 249 test_filter = ':'.join(shard_test_list) + self.gtest_filter |
244 return SingleTestRunner(device, self.test_suite, | 250 return SingleTestRunner(device, self.test_suite, |
245 test_filter, self.test_arguments, self.timeout, | 251 test_filter, self.test_arguments, self.timeout, |
246 self.rebaseline, self.performance_test, | 252 self.rebaseline, self.performance_test, |
247 self.cleanup_test_files, self.tool, index, | 253 self.cleanup_test_files, self.tool, index, |
248 not not self.log_dump_name, self.fast_and_loose) | 254 not not self.log_dump_name, self.fast_and_loose, |
| 255 self.build_type) |
249 | 256 |
250 def OnTestsCompleted(self, test_runners, test_results): | 257 def OnTestsCompleted(self, test_runners, test_results): |
251 """Notifies that we completed the tests.""" | 258 """Notifies that we completed the tests.""" |
252 test_results.LogFull('Unit test', os.path.basename(self.test_suite)) | 259 test_results.LogFull('Unit test', os.path.basename(self.test_suite)) |
253 PrintAnnotationForTestResults(test_results) | 260 PrintAnnotationForTestResults(test_results) |
254 if test_results.failed and self.rebaseline: | 261 if test_results.failed and self.rebaseline: |
255 test_runners[0].UpdateFilter(test_results.failed) | 262 test_runners[0].UpdateFilter(test_results.failed) |
256 if self.log_dump_name: | 263 if self.log_dump_name: |
257 # Zip all debug info outputs into a file named by log_dump_name. | 264 # Zip all debug info outputs into a file named by log_dump_name. |
258 debug_info.GTestDebugInfo.ZipAndCleanResults( | 265 debug_info.GTestDebugInfo.ZipAndCleanResults( |
259 os.path.join(constants.CHROME_DIR, 'out', 'Release', | 266 os.path.join(TestSuiteDir(self.build_type), 'debug_info_dumps'), |
260 'debug_info_dumps'), | |
261 self.log_dump_name) | 267 self.log_dump_name) |
262 | 268 |
263 | 269 |
264 def _RunATestSuite(options): | 270 def _RunATestSuite(options): |
265 """Run a single test suite. | 271 """Run a single test suite. |
266 | 272 |
267 Helper for Dispatch() to allow stop/restart of the emulator across | 273 Helper for Dispatch() to allow stop/restart of the emulator across |
268 test bundles. If using the emulator, we start it on entry and stop | 274 test bundles. If using the emulator, we start it on entry and stop |
269 it on exit. | 275 it on exit. |
270 | 276 |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
310 raise Exception('Failed to reset test server port.') | 316 raise Exception('Failed to reset test server port.') |
311 | 317 |
312 if options.performance_test or options.gtest_filter: | 318 if options.performance_test or options.gtest_filter: |
313 # These configuration can't be split in multiple devices. | 319 # These configuration can't be split in multiple devices. |
314 attached_devices = [attached_devices[0]] | 320 attached_devices = [attached_devices[0]] |
315 sharder = TestSharder(attached_devices, options.test_suite, | 321 sharder = TestSharder(attached_devices, options.test_suite, |
316 options.gtest_filter, options.test_arguments, | 322 options.gtest_filter, options.test_arguments, |
317 options.timeout, options.rebaseline, | 323 options.timeout, options.rebaseline, |
318 options.performance_test, | 324 options.performance_test, |
319 options.cleanup_test_files, options.tool, | 325 options.cleanup_test_files, options.tool, |
320 options.log_dump, options.fast_and_loose) | 326 options.log_dump, options.fast_and_loose, |
| 327 options.build_type) |
321 test_results = sharder.RunShardedTests() | 328 test_results = sharder.RunShardedTests() |
322 | 329 |
323 for buildbot_emulator in buildbot_emulators: | 330 for buildbot_emulator in buildbot_emulators: |
324 buildbot_emulator.Shutdown() | 331 buildbot_emulator.Shutdown() |
325 | 332 |
326 # Another chance if we timed out? At this point It is safe(r) to | 333 # Another chance if we timed out? At this point It is safe(r) to |
327 # run fast and loose since we just uploaded all the test data and | 334 # run fast and loose since we just uploaded all the test data and |
328 # binary. | 335 # binary. |
329 if test_results.timed_out and options.repeat: | 336 if test_results.timed_out and options.repeat: |
330 logging.critical('Timed out; repeating in fast_and_loose mode.') | 337 logging.critical('Timed out; repeating in fast_and_loose mode.') |
(...skipping 17 matching lines...) Expand all Loading... |
348 0 if successful, number of failing tests otherwise. | 355 0 if successful, number of failing tests otherwise. |
349 """ | 356 """ |
350 if options.test_suite == 'help': | 357 if options.test_suite == 'help': |
351 ListTestSuites() | 358 ListTestSuites() |
352 return 0 | 359 return 0 |
353 | 360 |
354 if options.use_xvfb: | 361 if options.use_xvfb: |
355 xvfb = Xvfb() | 362 xvfb = Xvfb() |
356 xvfb.Start() | 363 xvfb.Start() |
357 | 364 |
358 all_test_suites = FullyQualifiedTestSuites(options.exe, options.test_suite) | 365 all_test_suites = FullyQualifiedTestSuites(options.exe, options.test_suite, |
| 366 options.build_type) |
359 failures = 0 | 367 failures = 0 |
360 for suite in all_test_suites: | 368 for suite in all_test_suites: |
361 options.test_suite = suite | 369 options.test_suite = suite |
362 failures += _RunATestSuite(options) | 370 failures += _RunATestSuite(options) |
363 | 371 |
364 if options.use_xvfb: | 372 if options.use_xvfb: |
365 xvfb.Stop() | 373 xvfb.Stop() |
366 return failures | 374 return failures |
367 | 375 |
368 | 376 |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
415 'Don\'t use on bots by default!') | 423 'Don\'t use on bots by default!') |
416 option_parser.add_option('--repeat', dest='repeat', type='int', | 424 option_parser.add_option('--repeat', dest='repeat', type='int', |
417 default=2, | 425 default=2, |
418 help='Repeat count on test timeout') | 426 help='Repeat count on test timeout') |
419 option_parser.add_option('--exit_code', action='store_true', | 427 option_parser.add_option('--exit_code', action='store_true', |
420 help='If set, the exit code will be total number ' | 428 help='If set, the exit code will be total number ' |
421 'of failures.') | 429 'of failures.') |
422 option_parser.add_option('--exe', action='store_true', | 430 option_parser.add_option('--exe', action='store_true', |
423 help='If set, use the exe test runner instead of ' | 431 help='If set, use the exe test runner instead of ' |
424 'the APK.') | 432 'the APK.') |
| 433 |
425 options, args = option_parser.parse_args(argv) | 434 options, args = option_parser.parse_args(argv) |
426 if len(args) > 1: | 435 if len(args) > 1: |
427 print 'Unknown argument:', args[1:] | 436 print 'Unknown argument:', args[1:] |
428 option_parser.print_usage() | 437 option_parser.print_usage() |
429 sys.exit(1) | 438 sys.exit(1) |
430 run_tests_helper.SetLogLevel(options.verbose_count) | 439 run_tests_helper.SetLogLevel(options.verbose_count) |
431 emulator.DeleteAllTempAVDs() | 440 emulator.DeleteAllTempAVDs() |
432 failed_tests_count = Dispatch(options) | 441 failed_tests_count = Dispatch(options) |
433 | 442 |
434 # Failures of individual test suites are communicated by printing a | 443 # Failures of individual test suites are communicated by printing a |
435 # STEP_FAILURE message. | 444 # STEP_FAILURE message. |
436 # Returning a success exit status also prevents the buildbot from incorrectly | 445 # Returning a success exit status also prevents the buildbot from incorrectly |
437 # marking the last suite as failed if there were failures in other suites in | 446 # marking the last suite as failed if there were failures in other suites in |
438 # the batch (this happens because the exit status is a sum of all failures | 447 # the batch (this happens because the exit status is a sum of all failures |
439 # from all suites, but the buildbot associates the exit status only with the | 448 # from all suites, but the buildbot associates the exit status only with the |
440 # most recent step). | 449 # most recent step). |
441 if options.exit_code: | 450 if options.exit_code: |
442 return failed_tests_count | 451 return failed_tests_count |
443 return 0 | 452 return 0 |
444 | 453 |
445 | 454 |
446 if __name__ == '__main__': | 455 if __name__ == '__main__': |
447 sys.exit(main(sys.argv)) | 456 sys.exit(main(sys.argv)) |
OLD | NEW |