OLD | NEW |
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Implements test sharding logic.""" | 5 """Implements test sharding logic.""" |
6 | 6 |
7 import logging | 7 import logging |
8 import threading | 8 import threading |
9 | 9 |
10 from pylib import android_commands | 10 from pylib import android_commands |
| 11 from pylib import constants |
11 from pylib import forwarder | 12 from pylib import forwarder |
12 from pylib.utils import reraiser_thread | 13 from pylib.utils import reraiser_thread |
13 from pylib.utils import watchdog_timer | 14 from pylib.utils import watchdog_timer |
14 | 15 |
15 import base_test_result | 16 import base_test_result |
16 | 17 |
17 | 18 |
18 DEFAULT_TIMEOUT = 7 * 60 # seven minutes | 19 DEFAULT_TIMEOUT = 7 * 60 # seven minutes |
19 | 20 |
20 | 21 |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
85 try: | 86 try: |
86 return self._tests.pop(0) | 87 return self._tests.pop(0) |
87 except IndexError: | 88 except IndexError: |
88 # Another thread beat us to the avaliable test, wait again. | 89 # Another thread beat us to the avaliable test, wait again. |
89 self._item_avaliable_or_all_done.clear() | 90 self._item_avaliable_or_all_done.clear() |
90 | 91 |
91 def add(self, test): | 92 def add(self, test): |
92 """Add an test to the collection. | 93 """Add an test to the collection. |
93 | 94 |
94 Args: | 95 Args: |
95 item: A test to add. | 96 test: A test to add. |
96 """ | 97 """ |
97 with self._lock: | 98 with self._lock: |
98 self._tests.append(test) | 99 self._tests.append(test) |
99 self._item_avaliable_or_all_done.set() | 100 self._item_avaliable_or_all_done.set() |
100 self._tests_in_progress += 1 | 101 self._tests_in_progress += 1 |
101 | 102 |
102 def test_completed(self): | 103 def test_completed(self): |
103 """Indicate that a test has been fully handled.""" | 104 """Indicate that a test has been fully handled.""" |
104 with self._lock: | 105 with self._lock: |
105 self._tests_in_progress -= 1 | 106 self._tests_in_progress -= 1 |
106 if self._tests_in_progress == 0: | 107 if self._tests_in_progress == 0: |
107 # All tests have been handled, signal all waiting threads. | 108 # All tests have been handled, signal all waiting threads. |
108 self._item_avaliable_or_all_done.set() | 109 self._item_avaliable_or_all_done.set() |
109 | 110 |
110 def __iter__(self): | 111 def __iter__(self): |
111 """Iterate through tests in the collection until all have been handled.""" | 112 """Iterate through tests in the collection until all have been handled.""" |
112 while True: | 113 while True: |
113 r = self._pop() | 114 r = self._pop() |
114 if r is None: | 115 if r is None: |
115 break | 116 break |
116 yield r | 117 yield r |
117 | 118 |
118 | 119 |
119 def _RunTestsFromQueue(runner, test_collection, out_results, watcher, | 120 def _RunTestsFromQueue(runner, test_collection, out_results, watcher, |
120 num_retries): | 121 num_retries): |
121 """Runs tests from the test_collection until empty using the given runner. | 122 """Runs tests from the test_collection until empty using the given runner. |
122 | 123 |
123 Adds TestRunResults objects to the out_results list and may add tests to the | 124 Adds TestRunResults objects to the out_results list and may add tests to the |
124 out_retry list. | 125 out_retry list. |
125 | 126 |
126 Args: | 127 Args: |
127 runner: A TestRunner object used to run the tests. | 128 runner: A TestRunner object used to run the tests. |
128 test_collection: A _TestCollection from which to get _Test objects to run. | 129 test_collection: A _TestCollection from which to get _Test objects to run. |
129 out_results: A list to add TestRunResults to. | 130 out_results: A list to add TestRunResults to. |
130 watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout. | 131 watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout. |
(...skipping 12 matching lines...) Expand all Loading... |
143 if retry and test.tries <= num_retries: | 144 if retry and test.tries <= num_retries: |
144 # Retry non-passing results, only record passing results. | 145 # Retry non-passing results, only record passing results. |
145 pass_results = base_test_result.TestRunResults() | 146 pass_results = base_test_result.TestRunResults() |
146 pass_results.AddResults(result.GetPass()) | 147 pass_results.AddResults(result.GetPass()) |
147 out_results.append(pass_results) | 148 out_results.append(pass_results) |
148 logging.warning('Will retry test, try #%s.' % test.tries) | 149 logging.warning('Will retry test, try #%s.' % test.tries) |
149 test_collection.add(_Test(test=retry, tries=test.tries)) | 150 test_collection.add(_Test(test=retry, tries=test.tries)) |
150 else: | 151 else: |
151 # All tests passed or retry limit reached. Either way, record results. | 152 # All tests passed or retry limit reached. Either way, record results. |
152 out_results.append(result) | 153 out_results.append(result) |
153 except android_commands.errors.DeviceUnresponsiveError: | |
154 # Device is unresponsive, stop handling tests on this device and ensure | |
155 # current test gets runs by another device. Don't reraise this exception | |
156 # on the main thread. | |
157 test_collection.add(test) | |
158 return | |
159 except: | 154 except: |
160 # An unhandleable exception, ensure tests get run by another device and | 155 # An unhandleable exception, ensure tests get run by another device and |
161 # reraise this exception on the main thread. | 156 # reraise this exception on the main thread. |
162 test_collection.add(test) | 157 test_collection.add(test) |
163 raise | 158 raise |
164 finally: | 159 finally: |
165 # Retries count as separate tasks so always mark the popped test as done. | 160 # Retries count as separate tasks so always mark the popped test as done. |
166 test_collection.test_completed() | 161 test_collection.test_completed() |
167 | 162 |
168 | 163 |
(...skipping 23 matching lines...) Expand all Loading... |
192 def _RunAllTests(runners, tests, num_retries, timeout=None): | 187 def _RunAllTests(runners, tests, num_retries, timeout=None): |
193 """Run all tests using the given TestRunners. | 188 """Run all tests using the given TestRunners. |
194 | 189 |
195 Args: | 190 Args: |
196 runners: a list of TestRunner objects. | 191 runners: a list of TestRunner objects. |
197 tests: a list of Tests to run using the given TestRunners. | 192 tests: a list of Tests to run using the given TestRunners. |
198 num_retries: number of retries for a test. | 193 num_retries: number of retries for a test. |
199 timeout: watchdog timeout in seconds, defaults to the default timeout. | 194 timeout: watchdog timeout in seconds, defaults to the default timeout. |
200 | 195 |
201 Returns: | 196 Returns: |
202 A TestRunResults object. | 197 A tuple of (TestRunResults object, exit code) |
203 """ | 198 """ |
204 logging.warning('Running %s tests with %s test runners.' % | 199 logging.warning('Running %s tests with %s test runners.' % |
205 (len(tests), len(runners))) | 200 (len(tests), len(runners))) |
206 tests_collection = _TestCollection([_Test(t) for t in tests]) | 201 tests_collection = _TestCollection([_Test(t) for t in tests]) |
207 results = [] | 202 results = [] |
| 203 exit_code = 0 |
208 watcher = watchdog_timer.WatchdogTimer(timeout) | 204 watcher = watchdog_timer.WatchdogTimer(timeout) |
209 workers = reraiser_thread.ReraiserThreadGroup( | 205 workers = reraiser_thread.ReraiserThreadGroup( |
210 [reraiser_thread.ReraiserThread( | 206 [reraiser_thread.ReraiserThread( |
211 _RunTestsFromQueue, | 207 _RunTestsFromQueue, |
212 [r, tests_collection, results, watcher, num_retries], | 208 [r, tests_collection, results, watcher, num_retries], |
213 name=r.device[-4:]) | 209 name=r.device[-4:]) |
214 for r in runners]) | 210 for r in runners]) |
| 211 run_results = base_test_result.TestRunResults() |
215 workers.StartAll() | 212 workers.StartAll() |
216 workers.JoinAll(watcher) | 213 |
217 run_results = base_test_result.TestRunResults() | 214 # Catch DeviceUnresponsiveErrors and set a warning exit code |
| 215 try: |
| 216 workers.JoinAll(watcher) |
| 217 except android_commands.errors.DeviceUnresponsiveError as e: |
| 218 logging.error(e) |
| 219 exit_code = constants.WARNING_EXIT_CODE |
| 220 |
218 for r in results: | 221 for r in results: |
219 run_results.AddTestRunResults(r) | 222 run_results.AddTestRunResults(r) |
220 return run_results | 223 if not run_results.DidRunPass(): |
| 224 exit_code = constants.ERROR_EXIT_CODE |
| 225 return (run_results, exit_code) |
221 | 226 |
222 | 227 |
223 def _CreateRunners(runner_factory, devices, timeout=None): | 228 def _CreateRunners(runner_factory, devices, timeout=None): |
224 """Creates a test runner for each device and calls SetUp() in parallel. | 229 """Creates a test runner for each device and calls SetUp() in parallel. |
225 | 230 |
226 Note: if a device is unresponsive the corresponding TestRunner will not be | 231 Note: if a device is unresponsive the corresponding TestRunner will not be |
227 included in the returned list. | 232 included in the returned list. |
228 | 233 |
229 Args: | 234 Args: |
230 runner_factory: callable that takes a device and index and returns a | 235 runner_factory: callable that takes a device and index and returns a |
(...skipping 12 matching lines...) Expand all Loading... |
243 [runner_factory, d, runners, counter], | 248 [runner_factory, d, runners, counter], |
244 name=d[-4:]) | 249 name=d[-4:]) |
245 for d in devices]) | 250 for d in devices]) |
246 threads.StartAll() | 251 threads.StartAll() |
247 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) | 252 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) |
248 return runners | 253 return runners |
249 | 254 |
250 | 255 |
251 def _TearDownRunners(runners, timeout=None): | 256 def _TearDownRunners(runners, timeout=None): |
252 """Calls TearDown() for each test runner in parallel. | 257 """Calls TearDown() for each test runner in parallel. |
| 258 |
253 Args: | 259 Args: |
254 runners: a list of TestRunner objects. | 260 runners: a list of TestRunner objects. |
255 timeout: watchdog timeout in seconds, defaults to the default timeout. | 261 timeout: watchdog timeout in seconds, defaults to the default timeout. |
256 """ | 262 """ |
257 threads = reraiser_thread.ReraiserThreadGroup( | 263 threads = reraiser_thread.ReraiserThreadGroup( |
258 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:]) | 264 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:]) |
259 for r in runners]) | 265 for r in runners]) |
260 threads.StartAll() | 266 threads.StartAll() |
261 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) | 267 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) |
262 | 268 |
(...skipping 10 matching lines...) Expand all Loading... |
273 devices: list of attached device serial numbers as strings. | 279 devices: list of attached device serial numbers as strings. |
274 tests: list of tests to run. | 280 tests: list of tests to run. |
275 build_type: either 'Debug' or 'Release'. | 281 build_type: either 'Debug' or 'Release'. |
276 test_timeout: watchdog timeout in seconds for running tests, defaults to the | 282 test_timeout: watchdog timeout in seconds for running tests, defaults to the |
277 default timeout. | 283 default timeout. |
278 setup_timeout: watchdog timeout in seconds for creating and cleaning up | 284 setup_timeout: watchdog timeout in seconds for creating and cleaning up |
279 test runners, defaults to the default timeout. | 285 test runners, defaults to the default timeout. |
280 num_retries: number of retries for a test. | 286 num_retries: number of retries for a test. |
281 | 287 |
282 Returns: | 288 Returns: |
283 A base_test_result.TestRunResults object. | 289 A tuple of (base_test_result.TestRunResults object, exit code). |
284 """ | 290 """ |
285 if not tests: | 291 if not tests: |
286 logging.warning('No tests to run.') | 292 logging.error('No tests to run.') |
287 return base_test_result.TestRunResults() | 293 return (base_test_result.TestRunResults(), constants.ERROR_EXIT_CODE) |
288 | 294 |
289 logging.info('Will run %d tests: %s', len(tests), str(tests)) | 295 logging.info('Will run %d tests: %s', len(tests), str(tests)) |
290 forwarder.Forwarder.KillHost(build_type) | 296 forwarder.Forwarder.KillHost(build_type) |
291 runners = _CreateRunners(runner_factory, devices, setup_timeout) | 297 runners = _CreateRunners(runner_factory, devices, setup_timeout) |
292 try: | 298 try: |
293 return _RunAllTests(runners, tests, num_retries, test_timeout) | 299 return _RunAllTests(runners, tests, num_retries, test_timeout) |
294 finally: | 300 finally: |
295 try: | 301 try: |
296 _TearDownRunners(runners, setup_timeout) | 302 _TearDownRunners(runners, setup_timeout) |
297 except android_commands.errors.DeviceUnresponsiveError as e: | 303 except android_commands.errors.DeviceUnresponsiveError as e: |
298 logging.warning('Device unresponsive during TearDown: [%s]', e) | 304 logging.warning('Device unresponsive during TearDown: [%s]', e) |
299 finally: | 305 finally: |
300 forwarder.Forwarder.KillHost(build_type) | 306 forwarder.Forwarder.KillHost(build_type) |
OLD | NEW |