OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all types of tests from one unified interface. | 7 """Runs all types of tests from one unified interface. |
8 | 8 |
9 TODO(gkanwar): | 9 TODO(gkanwar): |
10 * Add options to run Monkey tests. | 10 * Add options to run Monkey tests. |
11 """ | 11 """ |
12 | 12 |
13 import collections | 13 import collections |
14 import optparse | 14 import optparse |
15 import os | 15 import os |
16 import shutil | 16 import shutil |
17 import sys | 17 import sys |
18 | 18 |
19 from pylib import cmd_helper | |
20 from pylib import constants | 19 from pylib import constants |
21 from pylib import ports | 20 from pylib import ports |
22 from pylib.base import base_test_result | 21 from pylib.base import base_test_result |
23 from pylib.base import test_dispatcher | 22 from pylib.base import test_dispatcher |
| 23 from pylib.gtest import gtest_config |
24 from pylib.gtest import setup as gtest_setup | 24 from pylib.gtest import setup as gtest_setup |
25 from pylib.gtest import gtest_config | 25 from pylib.gtest import test_options as gtest_test_options |
26 from pylib.host_driven import run_python_tests as python_dispatch | 26 from pylib.host_driven import run_python_tests as python_dispatch |
27 from pylib.instrumentation import setup as instrumentation_setup | 27 from pylib.instrumentation import setup as instrumentation_setup |
| 28 from pylib.instrumentation import test_options as instrumentation_test_options |
28 from pylib.uiautomator import setup as uiautomator_setup | 29 from pylib.uiautomator import setup as uiautomator_setup |
| 30 from pylib.uiautomator import test_options as uiautomator_test_options |
29 from pylib.utils import report_results | 31 from pylib.utils import report_results |
30 from pylib.utils import run_tests_helper | 32 from pylib.utils import run_tests_helper |
31 | 33 |
32 | 34 |
33 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') | 35 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') |
34 | 36 |
35 | 37 |
36 def AddBuildTypeOption(option_parser): | 38 def AddBuildTypeOption(option_parser): |
37 """Adds the build type option to |option_parser|.""" | 39 """Adds the build type option to |option_parser|.""" |
38 default_build_type = 'Debug' | 40 default_build_type = 'Debug' |
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
118 # TODO(gkanwar): Move these to Common Options once we have the plumbing | 120 # TODO(gkanwar): Move these to Common Options once we have the plumbing |
119 # in our other test types to handle these commands | 121 # in our other test types to handle these commands |
120 AddCommonOptions(option_parser) | 122 AddCommonOptions(option_parser) |
121 | 123 |
122 | 124 |
123 def ProcessGTestOptions(options): | 125 def ProcessGTestOptions(options): |
124 """Intercept test suite help to list test suites. | 126 """Intercept test suite help to list test suites. |
125 | 127 |
126 Args: | 128 Args: |
127 options: Command line options. | 129 options: Command line options. |
128 | |
129 Returns: | |
130 True if the command should continue. | |
131 """ | 130 """ |
132 if options.suite_name == 'help': | 131 if options.suite_name == 'help': |
133 print 'Available test suites are:' | 132 print 'Available test suites are:' |
134 for test_suite in (gtest_config.STABLE_TEST_SUITES + | 133 for test_suite in (gtest_config.STABLE_TEST_SUITES + |
135 gtest_config.EXPERIMENTAL_TEST_SUITES): | 134 gtest_config.EXPERIMENTAL_TEST_SUITES): |
136 print test_suite | 135 print test_suite |
137 return False | 136 sys.exit(0) |
138 | 137 |
139 # Convert to a list, assuming all test suites if nothing was specified. | 138 # Convert to a list, assuming all test suites if nothing was specified. |
140 # TODO(gkanwar): Require having a test suite | 139 # TODO(gkanwar): Require having a test suite |
141 if options.suite_name: | 140 if options.suite_name: |
142 options.suite_name = [options.suite_name] | 141 options.suite_name = [options.suite_name] |
143 else: | 142 else: |
144 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] | 143 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] |
145 return True | |
146 | 144 |
147 | 145 |
148 def AddJavaTestOptions(option_parser): | 146 def AddJavaTestOptions(option_parser): |
149 """Adds the Java test options to |option_parser|.""" | 147 """Adds the Java test options to |option_parser|.""" |
150 | 148 |
151 option_parser.add_option('-f', '--test_filter', dest='test_filter', | 149 option_parser.add_option('-f', '--test_filter', dest='test_filter', |
152 help=('Test filter (if not fully qualified, ' | 150 help=('Test filter (if not fully qualified, ' |
153 'will run all matches).')) | 151 'will run all matches).')) |
154 option_parser.add_option( | 152 option_parser.add_option( |
155 '-A', '--annotation', dest='annotation_str', | 153 '-A', '--annotation', dest='annotation_str', |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
246 option_parser.add_option('-I', dest='install_apk', action='store_true', | 244 option_parser.add_option('-I', dest='install_apk', action='store_true', |
247 help='(DEPRECATED) Install the test apk.') | 245 help='(DEPRECATED) Install the test apk.') |
248 option_parser.add_option( | 246 option_parser.add_option( |
249 '--test-apk', dest='test_apk', | 247 '--test-apk', dest='test_apk', |
250 help=('The name of the apk containing the tests ' | 248 help=('The name of the apk containing the tests ' |
251 '(without the .apk extension; e.g. "ContentShellTest"). ' | 249 '(without the .apk extension; e.g. "ContentShellTest"). ' |
252 'Alternatively, this can be a full path to the apk.')) | 250 'Alternatively, this can be a full path to the apk.')) |
253 | 251 |
254 | 252 |
255 def ProcessInstrumentationOptions(options, error_func): | 253 def ProcessInstrumentationOptions(options, error_func): |
256 """Processes options/arguments and populate |options| with defaults.""" | 254 """Processes options/arguments and populate |options| with defaults. |
| 255 |
| 256 Args: |
| 257 options: optparse.Options object. |
| 258 error_func: Function to call with the error message in case of an error. |
| 259 |
| 260 Returns: |
| 261 An InstrumentationOptions named tuple which contains all options relevant to |
| 262 instrumentation tests. |
| 263 """ |
257 | 264 |
258 ProcessJavaTestOptions(options, error_func) | 265 ProcessJavaTestOptions(options, error_func) |
259 | 266 |
260 if not options.test_apk: | 267 if not options.test_apk: |
261 error_func('--test-apk must be specified.') | 268 error_func('--test-apk must be specified.') |
262 | 269 |
263 if os.path.exists(options.test_apk): | 270 if os.path.exists(options.test_apk): |
264 # The APK is fully qualified, assume the JAR lives along side. | 271 # The APK is fully qualified, assume the JAR lives along side. |
265 options.test_apk_path = options.test_apk | 272 options.test_apk_path = options.test_apk |
266 options.test_apk_jar_path = (os.path.splitext(options.test_apk_path)[0] + | 273 options.test_apk_jar_path = (os.path.splitext(options.test_apk_path)[0] + |
267 '.jar') | 274 '.jar') |
268 else: | 275 else: |
269 options.test_apk_path = os.path.join(_SDK_OUT_DIR, | 276 options.test_apk_path = os.path.join(_SDK_OUT_DIR, |
270 options.build_type, | 277 options.build_type, |
271 constants.SDK_BUILD_APKS_DIR, | 278 constants.SDK_BUILD_APKS_DIR, |
272 '%s.apk' % options.test_apk) | 279 '%s.apk' % options.test_apk) |
273 options.test_apk_jar_path = os.path.join( | 280 options.test_apk_jar_path = os.path.join( |
274 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_TEST_JAVALIB_DIR, | 281 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_TEST_JAVALIB_DIR, |
275 '%s.jar' % options.test_apk) | 282 '%s.jar' % options.test_apk) |
276 | 283 |
| 284 return instrumentation_test_options.InstrumentationOptions( |
| 285 options.build_type, |
| 286 options.tool, |
| 287 options.cleanup_test_files, |
| 288 options.push_deps, |
| 289 options.annotations, |
| 290 options.exclude_annotations, |
| 291 options.test_filter, |
| 292 options.test_data, |
| 293 options.save_perf_json, |
| 294 options.screenshot_failures, |
| 295 options.disable_assertions, |
| 296 options.wait_for_debugger, |
| 297 options.test_apk, |
| 298 options.test_apk_path, |
| 299 options.test_apk_jar_path) |
| 300 |
277 | 301 |
278 def AddUIAutomatorTestOptions(option_parser): | 302 def AddUIAutomatorTestOptions(option_parser): |
279 """Adds UI Automator test options to |option_parser|.""" | 303 """Adds UI Automator test options to |option_parser|.""" |
280 | 304 |
281 option_parser.usage = '%prog uiautomator [options]' | 305 option_parser.usage = '%prog uiautomator [options]' |
282 option_parser.command_list = [] | 306 option_parser.command_list = [] |
283 option_parser.example = ( | 307 option_parser.example = ( |
284 '%prog uiautomator --test-jar=chromium_testshell_uiautomator_tests' | 308 '%prog uiautomator --test-jar=chromium_testshell_uiautomator_tests' |
285 ' --package-name=org.chromium.chrome.testshell') | 309 ' --package-name=org.chromium.chrome.testshell') |
286 option_parser.add_option( | 310 option_parser.add_option( |
287 '--package-name', | 311 '--package-name', |
288 help='The package name used by the apk containing the application.') | 312 help='The package name used by the apk containing the application.') |
289 option_parser.add_option( | 313 option_parser.add_option( |
290 '--test-jar', dest='test_jar', | 314 '--test-jar', dest='test_jar', |
291 help=('The name of the dexed jar containing the tests (without the ' | 315 help=('The name of the dexed jar containing the tests (without the ' |
292 '.dex.jar extension). Alternatively, this can be a full path ' | 316 '.dex.jar extension). Alternatively, this can be a full path ' |
293 'to the jar.')) | 317 'to the jar.')) |
294 | 318 |
295 AddJavaTestOptions(option_parser) | 319 AddJavaTestOptions(option_parser) |
296 AddCommonOptions(option_parser) | 320 AddCommonOptions(option_parser) |
297 | 321 |
298 | 322 |
299 def ProcessUIAutomatorOptions(options, error_func): | 323 def ProcessUIAutomatorOptions(options, error_func): |
300 """Processes UIAutomator options/arguments.""" | 324 """Processes UIAutomator options/arguments. |
| 325 |
| 326 Args: |
| 327 options: optparse.Options object. |
| 328 error_func: Function to call with the error message in case of an error. |
| 329 |
| 330 Returns: |
| 331 A UIAutomatorOptions named tuple which contains all options relevant to |
| 332 instrumentation tests. |
| 333 """ |
301 | 334 |
302 ProcessJavaTestOptions(options, error_func) | 335 ProcessJavaTestOptions(options, error_func) |
303 | 336 |
304 if not options.package_name: | 337 if not options.package_name: |
305 error_func('--package-name must be specified.') | 338 error_func('--package-name must be specified.') |
306 | 339 |
307 if not options.test_jar: | 340 if not options.test_jar: |
308 error_func('--test-jar must be specified.') | 341 error_func('--test-jar must be specified.') |
309 | 342 |
310 if os.path.exists(options.test_jar): | 343 if os.path.exists(options.test_jar): |
311 # The dexed JAR is fully qualified, assume the info JAR lives along side. | 344 # The dexed JAR is fully qualified, assume the info JAR lives along side. |
312 options.uiautomator_jar = options.test_jar | 345 options.uiautomator_jar = options.test_jar |
313 else: | 346 else: |
314 options.uiautomator_jar = os.path.join( | 347 options.uiautomator_jar = os.path.join( |
315 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_JAVALIB_DIR, | 348 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_JAVALIB_DIR, |
316 '%s.dex.jar' % options.test_jar) | 349 '%s.dex.jar' % options.test_jar) |
317 options.uiautomator_info_jar = ( | 350 options.uiautomator_info_jar = ( |
318 options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] + | 351 options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] + |
319 '_java.jar') | 352 '_java.jar') |
320 | 353 |
| 354 return uiautomator_test_options.UIAutomatorOptions( |
| 355 options.build_type, |
| 356 options.tool, |
| 357 options.cleanup_test_files, |
| 358 options.push_deps, |
| 359 options.annotations, |
| 360 options.exclude_annotations, |
| 361 options.test_filter, |
| 362 options.test_data, |
| 363 options.save_perf_json, |
| 364 options.screenshot_failures, |
| 365 options.disable_assertions, |
| 366 options.uiautomator_jar, |
| 367 options.uiautomator_info_jar, |
| 368 options.package_name) |
| 369 |
321 | 370 |
322 def _RunGTests(options, error_func): | 371 def _RunGTests(options, error_func): |
323 """Subcommand of RunTestsCommands which runs gtests.""" | 372 """Subcommand of RunTestsCommands which runs gtests.""" |
324 if not ProcessGTestOptions(options): | 373 ProcessGTestOptions(options) |
325 return 0 | |
326 | 374 |
327 exit_code = 0 | 375 exit_code = 0 |
328 for suite_name in options.suite_name: | 376 for suite_name in options.suite_name: |
329 runner_factory, tests = gtest_setup.Setup( | 377 # TODO(gkanwar): Move this into ProcessGTestOptions once we require -s for |
330 suite_name, options.test_arguments, | 378 # the gtest command. |
331 options.timeout, options.cleanup_test_files, options.tool, | 379 gtest_options = gtest_test_options.GTestOptions( |
332 options.build_type, options.push_deps, options.test_filter) | 380 options.build_type, |
| 381 options.tool, |
| 382 options.cleanup_test_files, |
| 383 options.push_deps, |
| 384 options.test_filter, |
| 385 options.test_arguments, |
| 386 options.timeout, |
| 387 suite_name) |
| 388 runner_factory, tests = gtest_setup.Setup(gtest_options) |
333 | 389 |
334 results, test_exit_code = test_dispatcher.RunTests( | 390 results, test_exit_code = test_dispatcher.RunTests( |
335 tests, runner_factory, False, options.test_device, | 391 tests, runner_factory, False, options.test_device, |
336 shard=True, | 392 shard=True, |
337 build_type=options.build_type, | 393 build_type=options.build_type, |
338 test_timeout=None, | 394 test_timeout=None, |
339 num_retries=options.num_retries) | 395 num_retries=options.num_retries) |
340 | 396 |
341 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: | 397 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: |
342 exit_code = test_exit_code | 398 exit_code = test_exit_code |
343 | 399 |
344 report_results.LogFull( | 400 report_results.LogFull( |
345 results=results, | 401 results=results, |
346 test_type='Unit test', | 402 test_type='Unit test', |
347 test_package=suite_name, | 403 test_package=suite_name, |
348 build_type=options.build_type, | 404 build_type=options.build_type, |
349 flakiness_server=options.flakiness_dashboard_server) | 405 flakiness_server=options.flakiness_dashboard_server) |
350 | 406 |
351 if os.path.isdir(constants.ISOLATE_DEPS_DIR): | 407 if os.path.isdir(constants.ISOLATE_DEPS_DIR): |
352 shutil.rmtree(constants.ISOLATE_DEPS_DIR) | 408 shutil.rmtree(constants.ISOLATE_DEPS_DIR) |
353 | 409 |
354 return exit_code | 410 return exit_code |
355 | 411 |
356 | 412 |
357 def _RunInstrumentationTests(options, error_func): | 413 def _RunInstrumentationTests(options, error_func): |
358 """Subcommand of RunTestsCommands which runs instrumentation tests.""" | 414 """Subcommand of RunTestsCommands which runs instrumentation tests.""" |
359 ProcessInstrumentationOptions(options, error_func) | 415 instrumentation_options = ProcessInstrumentationOptions(options, error_func) |
360 | 416 |
361 results = base_test_result.TestRunResults() | 417 results = base_test_result.TestRunResults() |
362 exit_code = 0 | 418 exit_code = 0 |
363 | 419 |
364 if options.run_java_tests: | 420 if options.run_java_tests: |
365 runner_factory, tests = instrumentation_setup.Setup( | 421 runner_factory, tests = instrumentation_setup.Setup(instrumentation_options) |
366 options.test_apk_path, options.test_apk_jar_path, options.annotations, | |
367 options.exclude_annotations, options.test_filter, options.build_type, | |
368 options.test_data, options.save_perf_json, options.screenshot_failures, | |
369 options.tool, options.wait_for_debugger, options.disable_assertions, | |
370 options.push_deps, options.cleanup_test_files) | |
371 | 422 |
372 test_results, exit_code = test_dispatcher.RunTests( | 423 test_results, exit_code = test_dispatcher.RunTests( |
373 tests, runner_factory, options.wait_for_debugger, | 424 tests, runner_factory, options.wait_for_debugger, |
374 options.test_device, | 425 options.test_device, |
375 shard=True, | 426 shard=True, |
376 build_type=options.build_type, | 427 build_type=options.build_type, |
377 test_timeout=None, | 428 test_timeout=None, |
378 num_retries=options.num_retries) | 429 num_retries=options.num_retries) |
379 | 430 |
380 results.AddTestRunResults(test_results) | 431 results.AddTestRunResults(test_results) |
(...skipping 14 matching lines...) Expand all Loading... |
395 test_package=os.path.basename(options.test_apk), | 446 test_package=os.path.basename(options.test_apk), |
396 annotation=options.annotations, | 447 annotation=options.annotations, |
397 build_type=options.build_type, | 448 build_type=options.build_type, |
398 flakiness_server=options.flakiness_dashboard_server) | 449 flakiness_server=options.flakiness_dashboard_server) |
399 | 450 |
400 return exit_code | 451 return exit_code |
401 | 452 |
402 | 453 |
403 def _RunUIAutomatorTests(options, error_func): | 454 def _RunUIAutomatorTests(options, error_func): |
404 """Subcommand of RunTestsCommands which runs uiautomator tests.""" | 455 """Subcommand of RunTestsCommands which runs uiautomator tests.""" |
405 ProcessUIAutomatorOptions(options, error_func) | 456 uiautomator_options = ProcessUIAutomatorOptions(options, error_func) |
406 | 457 |
407 results = base_test_result.TestRunResults() | 458 results = base_test_result.TestRunResults() |
408 exit_code = 0 | 459 exit_code = 0 |
409 | 460 |
410 if options.run_java_tests: | 461 if options.run_java_tests: |
411 runner_factory, tests = uiautomator_setup.Setup( | 462 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) |
412 options.uiautomator_jar, options.uiautomator_info_jar, | |
413 options.annotations, options.exclude_annotations, options.test_filter, | |
414 options.package_name, options.build_type, options.test_data, | |
415 options.save_perf_json, options.screenshot_failures, options.tool, | |
416 options.disable_assertions, options.push_deps, | |
417 options.cleanup_test_files) | |
418 | 463 |
419 test_results, exit_code = test_dispatcher.RunTests( | 464 test_results, exit_code = test_dispatcher.RunTests( |
420 tests, runner_factory, False, options.test_device, | 465 tests, runner_factory, False, options.test_device, |
421 shard=True, | 466 shard=True, |
422 build_type=options.build_type, | 467 build_type=options.build_type, |
423 test_timeout=None, | 468 test_timeout=None, |
424 num_retries=options.num_retries) | 469 num_retries=options.num_retries) |
425 | 470 |
426 results.AddTestRunResults(test_results) | 471 results.AddTestRunResults(test_results) |
427 | 472 |
(...skipping 148 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
576 return 0 | 621 return 0 |
577 command = argv[1] | 622 command = argv[1] |
578 VALID_COMMANDS[command].add_options_func(option_parser) | 623 VALID_COMMANDS[command].add_options_func(option_parser) |
579 options, args = option_parser.parse_args(argv) | 624 options, args = option_parser.parse_args(argv) |
580 return VALID_COMMANDS[command].run_command_func( | 625 return VALID_COMMANDS[command].run_command_func( |
581 command, options, args, option_parser) | 626 command, options, args, option_parser) |
582 | 627 |
583 | 628 |
584 if __name__ == '__main__': | 629 if __name__ == '__main__': |
585 sys.exit(main(sys.argv)) | 630 sys.exit(main(sys.argv)) |
OLD | NEW |