OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 import collections | 6 import collections |
7 import glob | 7 import glob |
8 import multiprocessing | 8 import multiprocessing |
9 import os | 9 import os |
10 import shutil | 10 import shutil |
11 import sys | 11 import sys |
12 | 12 |
13 import bb_utils | 13 import bb_utils |
14 | 14 |
15 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) | 15 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) |
16 from pylib import android_commands | 16 from pylib import android_commands |
17 from pylib import buildbot_report | 17 from pylib import buildbot_report |
18 from pylib import constants | 18 from pylib import constants |
19 from pylib.gtest import gtest_config | 19 from pylib.gtest import gtest_config |
20 | 20 |
21 sys.path.append(os.path.join( | 21 sys.path.append(os.path.join( |
22 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner')) | 22 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner')) |
23 import errors | 23 import errors |
24 | 24 |
25 | 25 |
26 CHROME_SRC = constants.DIR_SOURCE_ROOT | 26 CHROME_SRC = constants.DIR_SOURCE_ROOT |
| 27 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat') |
27 | 28 |
28 # Describes an instrumation test suite: | 29 # Describes an instrumation test suite: |
29 # test: Name of test we're running. | 30 # test: Name of test we're running. |
30 # apk: apk to be installed. | 31 # apk: apk to be installed. |
31 # apk_package: package for the apk to be installed. | 32 # apk_package: package for the apk to be installed. |
32 # test_apk: apk to run tests on. | 33 # test_apk: apk to run tests on. |
33 # test_data: data folder in format destination:source. | 34 # test_data: data folder in format destination:source. |
| 35 # host_driven_root: The python test root directory. |
| 36 # annotation: Annotation of the tests to include. |
| 37 # exclude_annotation: The annotation of the tests to exclude. |
34 I_TEST = collections.namedtuple('InstrumentationTest', [ | 38 I_TEST = collections.namedtuple('InstrumentationTest', [ |
35 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root']) | 39 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root', |
| 40 'annotation', 'exclude_annotation', 'extra_flags']) |
| 41 |
| 42 def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None, |
| 43 annotation=None, exclude_annotation=None, extra_flags=None): |
| 44 return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root, |
| 45 annotation, exclude_annotation, extra_flags) |
36 | 46 |
37 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ | 47 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ |
38 I_TEST('ContentShell', | 48 I('ContentShell', |
39 'ContentShell.apk', | 49 'ContentShell.apk', |
40 'org.chromium.content_shell_apk', | 50 'org.chromium.content_shell_apk', |
41 'ContentShellTest', | 51 'ContentShellTest', |
42 'content:content/test/data/android/device_files', | 52 'content:content/test/data/android/device_files'), |
43 None), | 53 I('ChromiumTestShell', |
44 I_TEST('ChromiumTestShell', | 54 'ChromiumTestShell.apk', |
45 'ChromiumTestShell.apk', | 55 'org.chromium.chrome.testshell', |
46 'org.chromium.chrome.testshell', | 56 'ChromiumTestShellTest', |
47 'ChromiumTestShellTest', | 57 'chrome:chrome/test/data/android/device_files', |
48 'chrome:chrome/test/data/android/device_files', | 58 constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR), |
49 constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR), | 59 I('AndroidWebView', |
50 I_TEST('AndroidWebView', | 60 'AndroidWebView.apk', |
51 'AndroidWebView.apk', | 61 'org.chromium.android_webview.shell', |
52 'org.chromium.android_webview.shell', | 62 'AndroidWebViewTest', |
53 'AndroidWebViewTest', | 63 'webview:android_webview/test/data/device_files'), |
54 'webview:android_webview/test/data/device_files', | |
55 None), | |
56 ]) | 64 ]) |
57 | 65 |
58 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout']) | 66 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout']) |
59 | 67 |
60 RunCmd = bb_utils.RunCmd | 68 RunCmd = bb_utils.RunCmd |
61 | 69 |
62 | 70 |
63 # multiprocessing map_async requires a top-level function for pickle library. | 71 # multiprocessing map_async requires a top-level function for pickle library. |
64 def RebootDeviceSafe(device): | 72 def RebootDeviceSafe(device): |
65 """Reboot a device, wait for it to start, and squelch timeout exceptions.""" | 73 """Reboot a device, wait for it to start, and squelch timeout exceptions.""" |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
117 options: options object. | 125 options: options object. |
118 """ | 126 """ |
119 args = ['--verbose', '--num_retries=1'] | 127 args = ['--verbose', '--num_retries=1'] |
120 if options.target == 'Release': | 128 if options.target == 'Release': |
121 args.append('--release') | 129 args.append('--release') |
122 if options.asan: | 130 if options.asan: |
123 args.append('--tool=asan') | 131 args.append('--tool=asan') |
124 buildbot_report.PrintNamedStep(constants.BROWSERTEST_SUITE_NAME) | 132 buildbot_report.PrintNamedStep(constants.BROWSERTEST_SUITE_NAME) |
125 RunCmd(['build/android/run_browser_tests.py'] + args) | 133 RunCmd(['build/android/run_browser_tests.py'] + args) |
126 | 134 |
127 def RunChromeDriverTests(): | 135 def RunChromeDriverTests(_): |
128 """Run all the steps for running chromedriver tests.""" | 136 """Run all the steps for running chromedriver tests.""" |
129 buildbot_report.PrintNamedStep('chromedriver_annotation') | 137 buildbot_report.PrintNamedStep('chromedriver_annotation') |
130 RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py', | 138 RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py', |
131 '--android-package=%s' % constants.CHROMIUM_TEST_SHELL_PACKAGE]) | 139 '--android-package=%s' % constants.CHROMIUM_TEST_SHELL_PACKAGE]) |
132 | 140 |
133 def InstallApk(options, test, print_step=False): | 141 def InstallApk(options, test, print_step=False): |
134 """Install an apk to all phones. | 142 """Install an apk to all phones. |
135 | 143 |
136 Args: | 144 Args: |
137 options: options object | 145 options: options object |
(...skipping 23 matching lines...) Expand all Loading... |
161 '--verbose', '-I'] | 169 '--verbose', '-I'] |
162 if options.target == 'Release': | 170 if options.target == 'Release': |
163 args.append('--release') | 171 args.append('--release') |
164 if options.asan: | 172 if options.asan: |
165 args.append('--tool=asan') | 173 args.append('--tool=asan') |
166 if options.upload_to_flakiness_server: | 174 if options.upload_to_flakiness_server: |
167 args.append('--flakiness-dashboard-server=%s' % | 175 args.append('--flakiness-dashboard-server=%s' % |
168 constants.UPSTREAM_FLAKINESS_SERVER) | 176 constants.UPSTREAM_FLAKINESS_SERVER) |
169 if test.host_driven_root: | 177 if test.host_driven_root: |
170 args.append('--python_test_root=%s' % test.host_driven_root) | 178 args.append('--python_test_root=%s' % test.host_driven_root) |
| 179 if test.annotation: |
| 180 args.extend(['-A', test.annotation]) |
| 181 if test.exclude_annotation: |
| 182 args.extend(['-E', test.exclude_annotation]) |
| 183 if test.extra_flags: |
| 184 args.extend(test.extra_flags) |
171 | 185 |
172 RunCmd(['build/android/run_instrumentation_tests.py'] + args) | 186 RunCmd(['build/android/run_instrumentation_tests.py'] + args) |
173 | 187 |
174 | 188 |
175 def RunWebkitLint(target): | 189 def RunWebkitLint(target): |
176 """Lint WebKit's TestExpectation files.""" | 190 """Lint WebKit's TestExpectation files.""" |
177 buildbot_report.PrintNamedStep('webkit_lint') | 191 buildbot_report.PrintNamedStep('webkit_lint') |
178 RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py', | 192 RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py', |
179 '--lint-test-files', | 193 '--lint-test-files', |
180 '--chromium', | 194 '--chromium', |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
212 # TODO(dpranke): Remove this block after | 226 # TODO(dpranke): Remove this block after |
213 # https://codereview.chromium.org/12927002/ lands. | 227 # https://codereview.chromium.org/12927002/ lands. |
214 for f in options.factory_properties.get('additional_expectations_files', []): | 228 for f in options.factory_properties.get('additional_expectations_files', []): |
215 cmd_args.extend( | 229 cmd_args.extend( |
216 ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)]) | 230 ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)]) |
217 | 231 |
218 RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args, | 232 RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args, |
219 flunk_on_failure=False) | 233 flunk_on_failure=False) |
220 | 234 |
221 | 235 |
222 def MainTestWrapper(options): | 236 def SpawnLogcatMonitor(): |
| 237 shutil.rmtree(LOGCAT_DIR, ignore_errors=True) |
| 238 bb_utils.SpawnCmd([ |
| 239 os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_monitor.py'), |
| 240 LOGCAT_DIR]) |
| 241 |
| 242 # Wait for logcat_monitor to pull existing logcat |
| 243 RunCmd(['sleep', '5']) |
| 244 |
| 245 def ProvisionDevices(options): |
223 # Restart adb to work around bugs, sleep to wait for usb discovery. | 246 # Restart adb to work around bugs, sleep to wait for usb discovery. |
224 RunCmd(['adb', 'kill-server']) | 247 RunCmd(['adb', 'kill-server']) |
225 RunCmd(['adb', 'start-server']) | 248 RunCmd(['adb', 'start-server']) |
226 RunCmd(['sleep', '1']) | 249 RunCmd(['sleep', '1']) |
227 | 250 |
228 # Spawn logcat monitor | |
229 logcat_dir = os.path.join(CHROME_SRC, 'out/logcat') | |
230 shutil.rmtree(logcat_dir, ignore_errors=True) | |
231 bb_utils.SpawnCmd(['build/android/adb_logcat_monitor.py', logcat_dir]) | |
232 | |
233 # Wait for logcat_monitor to pull existing logcat | |
234 RunCmd(['sleep', '5']) | |
235 | |
236 # Provision devices | |
237 buildbot_report.PrintNamedStep('provision_devices') | 251 buildbot_report.PrintNamedStep('provision_devices') |
238 if options.reboot: | 252 if options.reboot: |
239 RebootDevices() | 253 RebootDevices() |
240 RunCmd(['build/android/provision_devices.py', '-t', options.target]) | 254 RunCmd(['build/android/provision_devices.py', '-t', options.target]) |
241 | 255 |
242 # Device check and alert emails | 256 |
| 257 def DeviceStatusCheck(_): |
243 buildbot_report.PrintNamedStep('device_status_check') | 258 buildbot_report.PrintNamedStep('device_status_check') |
244 RunCmd(['build/android/device_status_check.py'], halt_on_failure=True) | 259 RunCmd(['build/android/device_status_check.py'], halt_on_failure=True) |
245 | 260 |
246 if options.install: | |
247 test_obj = INSTRUMENTATION_TESTS[options.install] | |
248 InstallApk(options, test_obj, print_step=True) | |
249 | 261 |
250 if 'chromedriver' in options.test_filter: | 262 def GetDeviceSetupStepCmds(): |
251 RunChromeDriverTests() | 263 return [ |
252 if 'unit' in options.test_filter: | 264 ('provision_devices', ProvisionDevices), |
253 RunTestSuites(options, gtest_config.STABLE_TEST_SUITES) | 265 ('device_status_check', DeviceStatusCheck) |
254 if 'ui' in options.test_filter: | 266 ] |
255 for test in INSTRUMENTATION_TESTS.itervalues(): | |
256 RunInstrumentationSuite(options, test) | |
257 if 'webkit' in options.test_filter: | |
258 RunTestSuites(options, [ | |
259 gtest_config.Apk('webkit_unit_tests'), | |
260 ]) | |
261 RunWebkitLint(options.target) | |
262 if 'webkit_layout' in options.test_filter: | |
263 RunWebkitLayoutTests(options) | |
264 | 267 |
265 if options.experimental: | |
266 RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) | |
267 RunBrowserTestSuite(options) | |
268 | 268 |
| 269 def RunUnitTests(options): |
| 270 RunTestSuites(options, gtest_config.STABLE_TEST_SUITES) |
| 271 |
| 272 |
| 273 def RunInstrumentationTests(options): |
| 274 for test in INSTRUMENTATION_TESTS.itervalues(): |
| 275 RunInstrumentationSuite(options, test) |
| 276 |
| 277 |
| 278 def RunWebkitTests(options): |
| 279 RunTestSuites(options, [gtest_config.Apk('webkit_unit_tests')]) |
| 280 RunWebkitLint(options.target) |
| 281 |
| 282 |
| 283 def GetTestStepCmds(): |
| 284 return [ |
| 285 ('chromedriver', RunChromeDriverTests), |
| 286 ('unit', RunUnitTests), |
| 287 ('ui', RunInstrumentationTests), |
| 288 ('webkit', RunWebkitTests), |
| 289 ('webkit_layout', RunWebkitLayoutTests) |
| 290 ] |
| 291 |
| 292 |
| 293 def LogcatDump(options): |
269 # Print logcat, kill logcat monitor | 294 # Print logcat, kill logcat monitor |
270 buildbot_report.PrintNamedStep('logcat_dump') | 295 buildbot_report.PrintNamedStep('logcat_dump') |
271 RunCmd(['build/android/adb_logcat_printer.py', logcat_dir]) | 296 logcat_file = os.path.join(CHROME_SRC, 'out', options.target, 'full_log') |
| 297 with open(logcat_file, 'w') as f: |
| 298 RunCmd([ |
| 299 os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_printer.py'), |
| 300 LOGCAT_DIR], stdout=f) |
| 301 RunCmd(['cat', logcat_file]) |
272 | 302 |
| 303 |
| 304 def GenerateTestReport(options): |
273 buildbot_report.PrintNamedStep('test_report') | 305 buildbot_report.PrintNamedStep('test_report') |
274 for report in glob.glob( | 306 for report in glob.glob( |
275 os.path.join(CHROME_SRC, 'out', options.target, 'test_logs', '*.log')): | 307 os.path.join(CHROME_SRC, 'out', options.target, 'test_logs', '*.log')): |
276 RunCmd(['cat', report]) | 308 RunCmd(['cat', report]) |
277 os.remove(report) | 309 os.remove(report) |
278 | 310 |
279 | 311 |
280 def main(argv): | 312 def GetPostTestStepCmds(): |
| 313 return [ |
| 314 ('logcat_dump', LogcatDump), |
| 315 ('test_report', GenerateTestReport) |
| 316 ] |
| 317 |
| 318 |
| 319 def MainTestWrapper(options): |
| 320 # Spawn logcat monitor |
| 321 SpawnLogcatMonitor() |
| 322 |
| 323 # Run all device setup steps |
| 324 for _, cmd in GetDeviceSetupStepCmds(): |
| 325 cmd(options) |
| 326 |
| 327 if options.install: |
| 328 test_obj = INSTRUMENTATION_TESTS[options.install] |
| 329 InstallApk(options, test_obj, print_step=True) |
| 330 |
| 331 if options.test_filter: |
| 332 bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options) |
| 333 |
| 334 if options.experimental: |
| 335 RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) |
| 336 RunBrowserTestSuite(options) |
| 337 |
| 338 # Run all post test steps |
| 339 for _, cmd in GetPostTestStepCmds(): |
| 340 cmd(options) |
| 341 |
| 342 |
| 343 def GetDeviceStepsOptParser(): |
281 parser = bb_utils.GetParser() | 344 parser = bb_utils.GetParser() |
282 parser.add_option('--experimental', action='store_true', | 345 parser.add_option('--experimental', action='store_true', |
283 help='Run experiemental tests') | 346 help='Run experiemental tests') |
284 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], | 347 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], |
285 action='append', | 348 action='append', |
286 help=('Run a test suite. Test suites: "%s"' % | 349 help=('Run a test suite. Test suites: "%s"' % |
287 '", "'.join(VALID_TESTS))) | 350 '", "'.join(VALID_TESTS))) |
288 parser.add_option('--asan', action='store_true', help='Run tests with asan.') | 351 parser.add_option('--asan', action='store_true', help='Run tests with asan.') |
289 parser.add_option('--install', metavar='<apk name>', | 352 parser.add_option('--install', metavar='<apk name>', |
290 help='Install an apk by name') | 353 help='Install an apk by name') |
291 parser.add_option('--reboot', action='store_true', | 354 parser.add_option('--reboot', action='store_true', |
292 help='Reboot devices before running tests') | 355 help='Reboot devices before running tests') |
293 parser.add_option('--upload-to-flakiness-server', action='store_true', | 356 parser.add_option('--upload-to-flakiness-server', action='store_true', |
294 help='Upload the results to the flakiness dashboard.') | 357 help='Upload the results to the flakiness dashboard.') |
295 parser.add_option( | 358 parser.add_option( |
296 '--auto-reconnect', action='store_true', | 359 '--auto-reconnect', action='store_true', |
297 help='Push script to device which restarts adbd on disconnections.') | 360 help='Push script to device which restarts adbd on disconnections.') |
| 361 parser.add_option( |
| 362 '--logcat-dump-output', |
| 363 help='The logcat dump output will be "tee"-ed into this file') |
| 364 |
| 365 return parser |
| 366 |
| 367 |
| 368 def main(argv): |
| 369 parser = GetDeviceStepsOptParser() |
298 options, args = parser.parse_args(argv[1:]) | 370 options, args = parser.parse_args(argv[1:]) |
299 | 371 |
300 if args: | 372 if args: |
301 return sys.exit('Unused args %s' % args) | 373 return sys.exit('Unused args %s' % args) |
302 | 374 |
303 unknown_tests = set(options.test_filter) - VALID_TESTS | 375 unknown_tests = set(options.test_filter) - VALID_TESTS |
304 if unknown_tests: | 376 if unknown_tests: |
305 return sys.exit('Unknown tests %s' % list(unknown_tests)) | 377 return sys.exit('Unknown tests %s' % list(unknown_tests)) |
306 | 378 |
307 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) | 379 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) |
308 | 380 |
309 MainTestWrapper(options) | 381 MainTestWrapper(options) |
310 | 382 |
311 | 383 |
312 if __name__ == '__main__': | 384 if __name__ == '__main__': |
313 sys.exit(main(sys.argv)) | 385 sys.exit(main(sys.argv)) |
OLD | NEW |