OLD | NEW |
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Dispatches GTests.""" | 5 """Dispatches GTests.""" |
6 | 6 |
7 import copy | 7 import copy |
8 import fnmatch | 8 import fnmatch |
9 import logging | 9 import logging |
10 import os | 10 import os |
11 | 11 |
12 from pylib import android_commands | 12 from pylib import android_commands |
13 from pylib import cmd_helper | 13 from pylib import cmd_helper |
14 from pylib import constants | 14 from pylib import constants |
15 from pylib import ports | 15 from pylib import ports |
| 16 from pylib.base import base_test_result |
16 from pylib.base import shard | 17 from pylib.base import shard |
17 from pylib.utils import emulator | 18 from pylib.utils import emulator |
18 from pylib.utils import report_results | 19 from pylib.utils import report_results |
19 from pylib.utils import xvfb | 20 from pylib.utils import xvfb |
20 | 21 |
21 import gtest_config | 22 import gtest_config |
22 import test_runner | 23 import test_runner |
23 | 24 |
24 | 25 |
25 def _FullyQualifiedTestSuites(exe, option_test_suite, build_type): | 26 def _FullyQualifiedTestSuites(exe, option_test_suite, build_type): |
26 """Get a list of absolute paths to test suite targets. | 27 """Get a list of absolute paths to test suite targets. |
27 | 28 |
28 Args: | 29 Args: |
29 exe: if True, use the executable-based test runner. | 30 exe: if True, use the executable-based test runner. |
30 option_test_suite: the test_suite specified as an option. | 31 option_test_suite: the test_suite specified as an option. |
31 build_type: 'Release' or 'Debug'. | 32 build_type: 'Release' or 'Debug'. |
32 | 33 |
33 Returns: | 34 Returns: |
34 A list of tuples containing the suite and absolute path. | 35 A list of tuples containing the suite and absolute path. |
35 Ex. ('content_unittests', | 36 Ex. ('content_unittests', |
36 '/tmp/chrome/src/out/Debug/content_unittests_apk/' | 37 '/tmp/chrome/src/out/Debug/content_unittests_apk/' |
37 'content_unittests-debug.apk') | 38 'content_unittests-debug.apk') |
| 39 |
| 40 Raises: |
| 41 Exception: If test suite not found. |
38 """ | 42 """ |
39 def GetQualifiedSuite(suite): | 43 def GetQualifiedSuite(suite): |
40 if suite.is_suite_exe: | 44 if suite.is_suite_exe: |
41 relpath = suite.name | 45 relpath = suite.name |
42 else: | 46 else: |
43 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk | 47 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk |
44 relpath = os.path.join(suite.name + '_apk', suite.name + '-debug.apk') | 48 relpath = os.path.join(suite.name + '_apk', suite.name + '-debug.apk') |
45 return suite.name, os.path.join(test_suite_dir, relpath) | 49 return suite.name, os.path.join(test_suite_dir, relpath) |
46 | 50 |
47 test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type) | 51 test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type) |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
86 Obtains a list of enabled tests from the test package on the device, | 90 Obtains a list of enabled tests from the test package on the device, |
87 then filters it again using the disabled list on the host. | 91 then filters it again using the disabled list on the host. |
88 | 92 |
89 Args: | 93 Args: |
90 runner_factory: callable that takes a devices and returns a TestRunner. | 94 runner_factory: callable that takes a devices and returns a TestRunner. |
91 devices: list of devices. | 95 devices: list of devices. |
92 | 96 |
93 Returns: | 97 Returns: |
94 List of all enabled tests. | 98 List of all enabled tests. |
95 | 99 |
96 Raises Exception if all devices failed. | 100 Raises: |
| 101 Exception: If no devices available. |
97 """ | 102 """ |
98 for device in devices: | 103 for device in devices: |
99 try: | 104 try: |
100 logging.info('Obtaining tests from %s', device) | 105 logging.info('Obtaining tests from %s', device) |
101 runner = runner_factory(device, 0) | 106 runner = runner_factory(device, 0) |
102 return GetTestsFromDevice(runner) | 107 return GetTestsFromDevice(runner) |
103 except Exception as e: | 108 except Exception as e: |
104 logging.warning('Failed obtaining tests from %s with exception: %s', | 109 logging.warning('Failed obtaining tests from %s with exception: %s', |
105 device, e) | 110 device, e) |
106 raise Exception('No device available to get the list of tests.') | 111 raise Exception('No device available to get the list of tests.') |
107 | 112 |
108 | 113 |
109 def _RunATestSuite(options, suite_name): | 114 def _RunATestSuite(options, suite_name): |
110 """Run a single test suite. | 115 """Run a single test suite. |
111 | 116 |
112 Helper for Dispatch() to allow stop/restart of the emulator across | 117 Helper for Dispatch() to allow stop/restart of the emulator across |
113 test bundles. If using the emulator, we start it on entry and stop | 118 test bundles. If using the emulator, we start it on entry and stop |
114 it on exit. | 119 it on exit. |
115 | 120 |
116 Args: | 121 Args: |
117 options: options for running the tests. | 122 options: options for running the tests. |
118 suite_name: name of the test suite being run. | 123 suite_name: name of the test suite being run. |
119 | 124 |
120 Returns: | 125 Returns: |
121 0 if successful, number of failing tests otherwise. | 126 A tuple of (base_test_result.TestRunResult object, exit code). |
| 127 |
| 128 Raises: |
| 129 Exception: For various reasons including device failure or failing to reset |
| 130 the test server port. |
122 """ | 131 """ |
123 step_name = os.path.basename(options.test_suite).replace('-debug.apk', '') | |
124 attached_devices = [] | 132 attached_devices = [] |
125 buildbot_emulators = [] | 133 buildbot_emulators = [] |
126 | 134 |
127 if options.use_emulator: | 135 if options.use_emulator: |
128 buildbot_emulators = emulator.LaunchEmulators(options.emulator_count, | 136 buildbot_emulators = emulator.LaunchEmulators(options.emulator_count, |
129 options.abi, | 137 options.abi, |
130 wait_for_boot=True) | 138 wait_for_boot=True) |
131 attached_devices = [e.device for e in buildbot_emulators] | 139 attached_devices = [e.device for e in buildbot_emulators] |
132 elif options.test_device: | 140 elif options.test_device: |
133 attached_devices = [options.test_device] | 141 attached_devices = [options.test_device] |
(...skipping 27 matching lines...) Expand all Loading... |
161 # Get tests and split them up based on the number of devices. | 169 # Get tests and split them up based on the number of devices. |
162 if options.test_filter: | 170 if options.test_filter: |
163 all_tests = [t for t in options.test_filter.split(':') if t] | 171 all_tests = [t for t in options.test_filter.split(':') if t] |
164 else: | 172 else: |
165 all_tests = GetAllEnabledTests(RunnerFactory, attached_devices) | 173 all_tests = GetAllEnabledTests(RunnerFactory, attached_devices) |
166 num_devices = len(attached_devices) | 174 num_devices = len(attached_devices) |
167 tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)] | 175 tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)] |
168 tests = [t for t in tests if t] | 176 tests = [t for t in tests if t] |
169 | 177 |
170 # Run tests. | 178 # Run tests. |
171 test_results = shard.ShardAndRunTests(RunnerFactory, attached_devices, tests, | 179 test_results, exit_code = shard.ShardAndRunTests( |
172 options.build_type, test_timeout=None, | 180 RunnerFactory, attached_devices, tests, options.build_type, |
173 num_retries=options.num_retries) | 181 test_timeout=None, num_retries=options.num_retries) |
174 | 182 |
175 report_results.LogFull( | 183 report_results.LogFull( |
176 results=test_results, | 184 results=test_results, |
177 test_type='Unit test', | 185 test_type='Unit test', |
178 test_package=suite_name, | 186 test_package=suite_name, |
179 build_type=options.build_type, | 187 build_type=options.build_type, |
180 flakiness_server=options.flakiness_dashboard_server) | 188 flakiness_server=options.flakiness_dashboard_server) |
181 report_results.PrintAnnotation(test_results) | |
182 | 189 |
183 for buildbot_emulator in buildbot_emulators: | 190 for buildbot_emulator in buildbot_emulators: |
184 buildbot_emulator.Shutdown() | 191 buildbot_emulator.Shutdown() |
185 | 192 |
186 return len(test_results.GetNotPass()) | 193 return (test_results, exit_code) |
187 | 194 |
188 | 195 |
189 def _ListTestSuites(): | 196 def _ListTestSuites(): |
190 """Display a list of available test suites.""" | 197 """Display a list of available test suites.""" |
191 print 'Available test suites are:' | 198 print 'Available test suites are:' |
192 for test_suite in gtest_config.STABLE_TEST_SUITES: | 199 for test_suite in gtest_config.STABLE_TEST_SUITES: |
193 print test_suite | 200 print test_suite |
194 | 201 |
195 | 202 |
196 def Dispatch(options): | 203 def Dispatch(options): |
197 """Dispatches the tests, sharding if possible. | 204 """Dispatches the tests, sharding if possible. |
198 | 205 |
199 If options.use_emulator is True, all tests will be run in new emulator | 206 If options.use_emulator is True, all tests will be run in new emulator |
200 instance. | 207 instance. |
201 | 208 |
202 Args: | 209 Args: |
203 options: options for running the tests. | 210 options: options for running the tests. |
204 | 211 |
205 Returns: | 212 Returns: |
206 0 if successful, number of failing tests otherwise. | 213 base_test_result.TestRunResults object with the results of running the tests |
207 """ | 214 """ |
208 if options.test_suite == 'help': | 215 if options.test_suite == 'help': |
209 _ListTestSuites() | 216 _ListTestSuites() |
210 return 0 | 217 return 0 |
211 | 218 |
212 if options.use_xvfb: | 219 if options.use_xvfb: |
213 framebuffer = xvfb.Xvfb() | 220 framebuffer = xvfb.Xvfb() |
214 framebuffer.Start() | 221 framebuffer.Start() |
215 | 222 |
216 all_test_suites = _FullyQualifiedTestSuites(options.exe, options.test_suite, | 223 all_test_suites = _FullyQualifiedTestSuites(options.exe, options.test_suite, |
217 options.build_type) | 224 options.build_type) |
218 failures = 0 | 225 results = base_test_result.TestRunResults() |
| 226 exit_code = 0 |
219 for suite_name, suite_path in all_test_suites: | 227 for suite_name, suite_path in all_test_suites: |
220 # Give each test suite its own copy of options. | 228 # Give each test suite its own copy of options. |
221 test_options = copy.deepcopy(options) | 229 test_options = copy.deepcopy(options) |
222 test_options.test_suite = suite_path | 230 test_options.test_suite = suite_path |
223 failures += _RunATestSuite(test_options, suite_name) | 231 test_results, test_exit_code = _RunATestSuite(test_options, suite_name) |
| 232 results.AddTestRunResults(test_results) |
| 233 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: |
| 234 exit_code = test_exit_code |
224 | 235 |
225 if options.use_xvfb: | 236 if options.use_xvfb: |
226 framebuffer.Stop() | 237 framebuffer.Stop() |
227 return failures | 238 |
| 239 return (results, exit_code) |
OLD | NEW |