OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all types of tests from one unified interface. | 7 """Runs all types of tests from one unified interface. |
8 | 8 |
9 TODO(gkanwar): | 9 TODO(gkanwar): |
10 * Add options to run Monkey tests. | 10 * Add options to run Monkey tests. |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
101 '(use --tool help to list them)')) | 101 '(use --tool help to list them)')) |
102 option_parser.add_option('--flakiness-dashboard-server', | 102 option_parser.add_option('--flakiness-dashboard-server', |
103 dest='flakiness_dashboard_server', | 103 dest='flakiness_dashboard_server', |
104 help=('Address of the server that is hosting the ' | 104 help=('Address of the server that is hosting the ' |
105 'Chrome for Android flakiness dashboard.')) | 105 'Chrome for Android flakiness dashboard.')) |
106 option_parser.add_option('--skip-deps-push', dest='push_deps', | 106 option_parser.add_option('--skip-deps-push', dest='push_deps', |
107 action='store_false', default=True, | 107 action='store_false', default=True, |
108 help=('Do not push dependencies to the device. ' | 108 help=('Do not push dependencies to the device. ' |
109 'Use this at own risk for speeding up test ' | 109 'Use this at own risk for speeding up test ' |
110 'execution on local machine.')) | 110 'execution on local machine.')) |
111 # TODO(gkanwar): This option is deprecated. Remove it in the future. | |
112 option_parser.add_option('--exit-code', action='store_true', | |
113 help=('(DEPRECATED) If set, the exit code will be ' | |
114 'total number of failures.')) | |
115 # TODO(gkanwar): This option is deprecated. It is currently used to run tests | |
116 # with the FlakyTest annotation to prevent the bots going red downstream. We | |
117 # should instead use exit codes and let the Buildbot scripts deal with test | |
118 # failures appropriately. See crbug.com/170477. | |
119 option_parser.add_option('--buildbot-step-failure', | |
120 action='store_true', | |
121 help=('(DEPRECATED) If present, will set the ' | |
122 'buildbot status as STEP_FAILURE, otherwise ' | |
123 'as STEP_WARNINGS when test(s) fail.')) | |
124 option_parser.add_option('-d', '--device', dest='test_device', | 111 option_parser.add_option('-d', '--device', dest='test_device', |
125 help=('Target device for the test suite ' | 112 help=('Target device for the test suite ' |
126 'to run on.')) | 113 'to run on.')) |
127 | 114 |
128 | 115 |
129 def ProcessCommonOptions(options): | 116 def ProcessCommonOptions(options): |
130 """Processes and handles all common options.""" | 117 """Processes and handles all common options.""" |
131 if options.out_directory: | 118 if options.out_directory: |
132 cmd_helper.OutDirectory.set(options.out_directory) | 119 cmd_helper.OutDirectory.set(options.out_directory) |
133 run_tests_helper.SetLogLevel(options.verbose_count) | 120 run_tests_helper.SetLogLevel(options.verbose_count) |
(...skipping 235 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
369 | 356 |
370 Args: | 357 Args: |
371 command: String indicating the command that was received to trigger | 358 command: String indicating the command that was received to trigger |
372 this function. | 359 this function. |
373 options: optparse options dictionary. | 360 options: optparse options dictionary. |
374 args: List of extra args from optparse. | 361 args: List of extra args from optparse. |
375 option_parser: optparse.OptionParser object. | 362 option_parser: optparse.OptionParser object. |
376 | 363 |
377 Returns: | 364 Returns: |
378 Integer indicated exit code. | 365 Integer indicated exit code. |
| 366 |
| 367 Raises: |
| 368 Exception: Unknown command name passed in, or an exception from an |
| 369 individual test runner. |
379 """ | 370 """ |
380 | 371 |
381 ProcessCommonOptions(options) | 372 ProcessCommonOptions(options) |
382 | 373 |
383 total_failed = 0 | |
384 if command == 'gtest': | 374 if command == 'gtest': |
385 # TODO(gkanwar): See the emulator TODO above -- this call should either go | 375 # TODO(gkanwar): See the emulator TODO above -- this call should either go |
386 # away or become generalized. | 376 # away or become generalized. |
387 ProcessEmulatorOptions(options) | 377 ProcessEmulatorOptions(options) |
388 total_failed = gtest_dispatch.Dispatch(options) | 378 results, exit_code = gtest_dispatch.Dispatch(options) |
389 elif command == 'content_browsertests': | 379 elif command == 'content_browsertests': |
390 total_failed = browsertests_dispatch.Dispatch(options) | 380 results, exit_code = browsertests_dispatch.Dispatch(options) |
391 elif command == 'instrumentation': | 381 elif command == 'instrumentation': |
392 ProcessInstrumentationOptions(options, option_parser.error) | 382 ProcessInstrumentationOptions(options, option_parser.error) |
393 results = base_test_result.TestRunResults() | 383 results = base_test_result.TestRunResults() |
| 384 exit_code = 0 |
394 if options.run_java_tests: | 385 if options.run_java_tests: |
395 results.AddTestRunResults(instrumentation_dispatch.Dispatch(options)) | 386 test_results, exit_code = instrumentation_dispatch.Dispatch(options) |
| 387 results.AddTestRunResults(test_results) |
396 if options.run_python_tests: | 388 if options.run_python_tests: |
397 results.AddTestRunResults(python_dispatch.DispatchPythonTests(options)) | 389 test_results, test_exit_code = (python_dispatch. |
| 390 DispatchPythonTests(options)) |
| 391 results.AddTestRunResults(test_results) |
| 392 # Only allow exit code escalation |
| 393 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: |
| 394 exit_code = test_exit_code |
398 report_results.LogFull( | 395 report_results.LogFull( |
399 results=results, | 396 results=results, |
400 test_type='Instrumentation', | 397 test_type='Instrumentation', |
401 test_package=os.path.basename(options.test_apk), | 398 test_package=os.path.basename(options.test_apk), |
402 annotation=options.annotations, | 399 annotation=options.annotations, |
403 build_type=options.build_type, | 400 build_type=options.build_type, |
404 flakiness_server=options.flakiness_dashboard_server) | 401 flakiness_server=options.flakiness_dashboard_server) |
405 total_failed += len(results.GetNotPass()) | |
406 elif command == 'uiautomator': | 402 elif command == 'uiautomator': |
407 ProcessUIAutomatorOptions(options, option_parser.error) | 403 ProcessUIAutomatorOptions(options, option_parser.error) |
408 results = base_test_result.TestRunResults() | 404 results = base_test_result.TestRunResults() |
| 405 exit_code = 0 |
409 if options.run_java_tests: | 406 if options.run_java_tests: |
410 results.AddTestRunResults(uiautomator_dispatch.Dispatch(options)) | 407 test_results, exit_code = uiautomator_dispatch.Dispatch(options) |
| 408 results.AddTestRunResults(test_results) |
411 if options.run_python_tests: | 409 if options.run_python_tests: |
412 results.AddTestRunResults(python_dispatch.Dispatch(options)) | 410 test_results, test_exit_code = (python_dispatch. |
| 411 DispatchPythonTests(options)) |
| 412 results.AddTestRunResults(test_results) |
| 413 # Only allow exit code escalation |
| 414 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: |
| 415 exit_code = test_exit_code |
413 report_results.LogFull( | 416 report_results.LogFull( |
414 results=results, | 417 results=results, |
415 test_type='UIAutomator', | 418 test_type='UIAutomator', |
416 test_package=os.path.basename(options.test_jar), | 419 test_package=os.path.basename(options.test_jar), |
417 annotation=options.annotations, | 420 annotation=options.annotations, |
418 build_type=options.build_type, | 421 build_type=options.build_type, |
419 flakiness_server=options.flakiness_dashboard_server) | 422 flakiness_server=options.flakiness_dashboard_server) |
420 total_failed += len(results.GetNotPass()) | |
421 else: | 423 else: |
422 raise Exception('Unknown test type state') | 424 raise Exception('Unknown test type state') |
423 | 425 |
424 return total_failed | 426 return exit_code |
425 | 427 |
426 | 428 |
427 def HelpCommand(command, options, args, option_parser): | 429 def HelpCommand(command, options, args, option_parser): |
428 """Display help for a certain command, or overall help. | 430 """Display help for a certain command, or overall help. |
429 | 431 |
430 Args: | 432 Args: |
431 command: String indicating the command that was received to trigger | 433 command: String indicating the command that was received to trigger |
432 this function. | 434 this function. |
433 options: optparse options dictionary. | 435 options: optparse options dictionary. |
434 args: List of extra args from optparse. | 436 args: List of extra args from optparse. |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
497 def get_command_list(self): | 499 def get_command_list(self): |
498 if self.command_list: | 500 if self.command_list: |
499 return '\nCommands:\n %s\n' % '\n '.join(sorted(self.command_list)) | 501 return '\nCommands:\n %s\n' % '\n '.join(sorted(self.command_list)) |
500 return '' | 502 return '' |
501 | 503 |
502 def get_example(self): | 504 def get_example(self): |
503 if self.example: | 505 if self.example: |
504 return '\nExample:\n %s\n' % self.example | 506 return '\nExample:\n %s\n' % self.example |
505 return '' | 507 return '' |
506 | 508 |
| 509 |
507 def main(argv): | 510 def main(argv): |
508 option_parser = CommandOptionParser( | 511 option_parser = CommandOptionParser( |
509 usage='Usage: %prog <command> [options]', | 512 usage='Usage: %prog <command> [options]', |
510 command_list=VALID_COMMANDS.keys()) | 513 command_list=VALID_COMMANDS.keys()) |
511 | 514 |
512 if len(argv) < 2 or argv[1] not in VALID_COMMANDS: | 515 if len(argv) < 2 or argv[1] not in VALID_COMMANDS: |
513 option_parser.print_help() | 516 option_parser.print_help() |
514 return 0 | 517 return 0 |
515 command = argv[1] | 518 command = argv[1] |
516 VALID_COMMANDS[command].add_options_func(option_parser) | 519 VALID_COMMANDS[command].add_options_func(option_parser) |
517 options, args = option_parser.parse_args(argv) | 520 options, args = option_parser.parse_args(argv) |
518 exit_code = VALID_COMMANDS[command].run_command_func( | 521 return VALID_COMMANDS[command].run_command_func( |
519 command, options, args, option_parser) | 522 command, options, args, option_parser) |
520 | 523 |
521 # Failures of individual test suites are communicated by printing a | |
522 # STEP_FAILURE message. | |
523 # Returning a success exit status also prevents the buildbot from incorrectly | |
524 # marking the last suite as failed if there were failures in other suites in | |
525 # the batch (this happens because the exit status is a sum of all failures | |
526 # from all suites, but the buildbot associates the exit status only with the | |
527 # most recent step). | |
528 return exit_code | |
529 | |
530 | 524 |
531 if __name__ == '__main__': | 525 if __name__ == '__main__': |
532 sys.exit(main(sys.argv)) | 526 sys.exit(main(sys.argv)) |
OLD | NEW |