OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged | 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged |
3 # | 3 # |
4 # Redistribution and use in source and binary forms, with or without | 4 # Redistribution and use in source and binary forms, with or without |
5 # modification, are permitted provided that the following conditions are | 5 # modification, are permitted provided that the following conditions are |
6 # met: | 6 # met: |
7 # | 7 # |
8 # * Redistributions of source code must retain the above copyright | 8 # * Redistributions of source code must retain the above copyright |
9 # notice, this list of conditions and the following disclaimer. | 9 # notice, this list of conditions and the following disclaimer. |
10 # * Redistributions in binary form must reproduce the above | 10 # * Redistributions in binary form must reproduce the above |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
74 self._port = port | 74 self._port = port |
75 self._filesystem = port.host.filesystem | 75 self._filesystem = port.host.filesystem |
76 self._options = options | 76 self._options = options |
77 self._printer = printer | 77 self._printer = printer |
78 self._expectations = None | 78 self._expectations = None |
79 | 79 |
80 self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR | 80 self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR |
81 self.PERF_SUBDIR = 'perf' | 81 self.PERF_SUBDIR = 'perf' |
82 self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR | 82 self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR |
83 self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests' | 83 self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests' |
84 | 84 self._http_server_started = False |
85 # disable wss server. need to install pyOpenSSL on buildbots. | 85 self._websockets_server_started = False |
86 # self._websocket_secure_server = websocket_server.PyWebSocket( | |
87 # options.results_directory, use_tls=True, port=9323) | |
88 | 86 |
89 self._results_directory = self._port.results_directory() | 87 self._results_directory = self._port.results_directory() |
90 self._finder = LayoutTestFinder(self._port, self._options) | 88 self._finder = LayoutTestFinder(self._port, self._options) |
91 self._runner = LayoutTestRunner(self._options, self._port, self._printer
, self._results_directory, self._test_is_slow) | 89 self._runner = LayoutTestRunner(self._options, self._port, self._printer
, self._results_directory, self._test_is_slow) |
92 | 90 |
93 def _collect_tests(self, args): | 91 def _collect_tests(self, args): |
94 return self._finder.find_tests(self._options, args) | 92 return self._finder.find_tests(self._options, args) |
95 | 93 |
96 def _is_http_test(self, test): | 94 def _is_http_test(self, test): |
97 return self.HTTP_SUBDIR in test or self._is_websocket_test(test) | 95 return self.HTTP_SUBDIR in test or self._is_websocket_test(test) |
(...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
190 | 188 |
191 if not self._set_up_run(tests_to_run): | 189 if not self._set_up_run(tests_to_run): |
192 return test_run_results.RunDetails(exit_code=-1) | 190 return test_run_results.RunDetails(exit_code=-1) |
193 | 191 |
194 # Don't retry failures if an explicit list of tests was passed in. | 192 # Don't retry failures if an explicit list of tests was passed in. |
195 if self._options.retry_failures is None: | 193 if self._options.retry_failures is None: |
196 should_retry_failures = len(paths) < len(test_names) | 194 should_retry_failures = len(paths) < len(test_names) |
197 else: | 195 else: |
198 should_retry_failures = self._options.retry_failures | 196 should_retry_failures = self._options.retry_failures |
199 | 197 |
| 198 |
200 start_time = time.time() | 199 start_time = time.time() |
201 enabled_pixel_tests_in_retry = False | 200 enabled_pixel_tests_in_retry = False |
202 try: | 201 try: |
| 202 self._start_servers(tests_to_run) |
| 203 |
203 initial_results = self._run_tests(tests_to_run, tests_to_skip, self.
_options.repeat_each, self._options.iterations, | 204 initial_results = self._run_tests(tests_to_run, tests_to_skip, self.
_options.repeat_each, self._options.iterations, |
204 int(self._options.child_processes), retrying=False) | 205 int(self._options.child_processes), retrying=False) |
205 | 206 |
206 tests_to_retry = self._tests_to_retry(initial_results, include_crash
es=self._port.should_retry_crashes()) | 207 tests_to_retry = self._tests_to_retry(initial_results, include_crash
es=self._port.should_retry_crashes()) |
207 if should_retry_failures and tests_to_retry and not initial_results.
interrupted: | 208 if should_retry_failures and tests_to_retry and not initial_results.
interrupted: |
208 enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed
() | 209 enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed
() |
209 | 210 |
210 _log.info('') | 211 _log.info('') |
211 _log.info("Retrying %d unexpected failure(s) ..." % len(tests_to
_retry)) | 212 _log.info("Retrying %d unexpected failure(s) ..." % len(tests_to
_retry)) |
212 _log.info('') | 213 _log.info('') |
213 retry_results = self._run_tests(tests_to_retry, tests_to_skip=se
t(), repeat_each=1, iterations=1, | 214 retry_results = self._run_tests(tests_to_retry, tests_to_skip=se
t(), repeat_each=1, iterations=1, |
214 num_workers=1, retrying=True) | 215 num_workers=1, retrying=True) |
215 | 216 |
216 if enabled_pixel_tests_in_retry: | 217 if enabled_pixel_tests_in_retry: |
217 self._options.pixel_tests = False | 218 self._options.pixel_tests = False |
218 else: | 219 else: |
219 retry_results = None | 220 retry_results = None |
220 finally: | 221 finally: |
| 222 self._stop_servers() |
221 self._clean_up_run() | 223 self._clean_up_run() |
222 | 224 |
223 end_time = time.time() | 225 end_time = time.time() |
224 | 226 |
225 # Some crash logs can take a long time to be written out so look | 227 # Some crash logs can take a long time to be written out so look |
226 # for new logs after the test run finishes. | 228 # for new logs after the test run finishes. |
227 _log.debug("looking for new crash logs") | 229 _log.debug("looking for new crash logs") |
228 self._look_for_new_crash_logs(initial_results, start_time) | 230 self._look_for_new_crash_logs(initial_results, start_time) |
229 if retry_results: | 231 if retry_results: |
230 self._look_for_new_crash_logs(retry_results, start_time) | 232 self._look_for_new_crash_logs(retry_results, start_time) |
231 | 233 |
232 _log.debug("summarizing results") | 234 _log.debug("summarizing results") |
233 summarized_full_results = test_run_results.summarize_results(self._port,
self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retr
y) | 235 summarized_full_results = test_run_results.summarize_results(self._port,
self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retr
y) |
234 summarized_failing_results = test_run_results.summarize_results(self._po
rt, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_r
etry, only_include_failing=True) | 236 summarized_failing_results = test_run_results.summarize_results(self._po
rt, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_r
etry, only_include_failing=True) |
235 self._printer.print_results(end_time - start_time, initial_results, summ
arized_failing_results) | 237 self._printer.print_results(end_time - start_time, initial_results, summ
arized_failing_results) |
236 | 238 |
237 exit_code = self._port.exit_code_from_summarized_results(summarized_fail
ing_results) | 239 exit_code = self._port.exit_code_from_summarized_results(summarized_fail
ing_results) |
238 if not self._options.dry_run: | 240 if not self._options.dry_run: |
239 self._write_json_files(summarized_full_results, summarized_failing_r
esults, initial_results) | 241 self._write_json_files(summarized_full_results, summarized_failing_r
esults, initial_results) |
240 self._upload_json_files() | 242 self._upload_json_files() |
241 | 243 |
242 results_path = self._filesystem.join(self._results_directory, "resul
ts.html") | 244 results_path = self._filesystem.join(self._results_directory, "resul
ts.html") |
243 self._copy_results_html_file(results_path) | 245 self._copy_results_html_file(results_path) |
244 if self._options.show_results and (exit_code or (self._options.full_
results_html and initial_results.total_failures)): | 246 if self._options.show_results and (exit_code or (self._options.full_
results_html and initial_results.total_failures)): |
245 self._port.show_results_html_file(results_path) | 247 self._port.show_results_html_file(results_path) |
246 | 248 |
247 return test_run_results.RunDetails(exit_code, summarized_full_results, s
ummarized_failing_results, initial_results, retry_results, enabled_pixel_tests_i
n_retry) | 249 return test_run_results.RunDetails(exit_code, summarized_full_results, s
ummarized_failing_results, initial_results, retry_results, enabled_pixel_tests_i
n_retry) |
248 | 250 |
249 def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations, n
um_workers, retrying): | 251 def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations, n
um_workers, retrying): |
250 needs_http = self._port.requires_http_server() or any(self._is_http_test
(test) for test in tests_to_run) | |
251 needs_websockets = any(self._is_websocket_test(test) for test in tests_t
o_run) | |
252 | 252 |
253 test_inputs = [] | 253 test_inputs = [] |
254 for _ in xrange(iterations): | 254 for _ in xrange(iterations): |
255 for test in tests_to_run: | 255 for test in tests_to_run: |
256 for _ in xrange(repeat_each): | 256 for _ in xrange(repeat_each): |
257 test_inputs.append(self._test_input_for_file(test)) | 257 test_inputs.append(self._test_input_for_file(test)) |
258 return self._runner.run_tests(self._expectations, test_inputs, tests_to_
skip, num_workers, needs_http, needs_websockets, retrying) | 258 return self._runner.run_tests(self._expectations, test_inputs, tests_to_
skip, num_workers, retrying) |
| 259 |
| 260 def _start_servers(self, tests_to_run): |
| 261 if self._port.requires_http_server() or any(self._is_http_test(test) for
test in tests_to_run): |
| 262 self._printer.write_update('Starting HTTP server ...') |
| 263 self._port.start_http_server(number_of_servers=(2 * self._options.ma
x_locked_shards)) |
| 264 self._http_server_started = True |
| 265 |
| 266 if any(self._is_websocket_test(test) for test in tests_to_run): |
| 267 self._printer.write_update('Starting WebSocket server ...') |
| 268 self._port.start_websocket_server() |
| 269 self._websockets_server_started = True |
| 270 |
| 271 def _stop_servers(self): |
| 272 if self._http_server_started: |
| 273 self._printer.write_update('Stopping HTTP server ...') |
| 274 self._http_server_started = False |
| 275 self._port.stop_http_server() |
| 276 if self._websockets_server_started: |
| 277 self._printer.write_update('Stopping WebSocket server ...') |
| 278 self._websockets_server_started = False |
| 279 self._port.stop_websocket_server() |
259 | 280 |
260 def _clean_up_run(self): | 281 def _clean_up_run(self): |
261 _log.debug("Flushing stdout") | 282 _log.debug("Flushing stdout") |
262 sys.stdout.flush() | 283 sys.stdout.flush() |
263 _log.debug("Flushing stderr") | 284 _log.debug("Flushing stderr") |
264 sys.stderr.flush() | 285 sys.stderr.flush() |
265 _log.debug("Stopping helper") | 286 _log.debug("Stopping helper") |
266 self._port.stop_helper() | 287 self._port.stop_helper() |
267 _log.debug("Cleaning up port") | 288 _log.debug("Cleaning up port") |
268 self._port.clean_up_test_run() | 289 self._port.clean_up_test_run() |
(...skipping 121 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
390 return int(worker_name.split('/')[1]) if worker_name else -1 | 411 return int(worker_name.split('/')[1]) if worker_name else -1 |
391 | 412 |
392 stats = {} | 413 stats = {} |
393 for result in initial_results.results_by_name.values(): | 414 for result in initial_results.results_by_name.values(): |
394 if result.type != test_expectations.SKIP: | 415 if result.type != test_expectations.SKIP: |
395 stats[result.test_name] = {'results': (_worker_number(result.wor
ker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int
(result.total_run_time * 1000))} | 416 stats[result.test_name] = {'results': (_worker_number(result.wor
ker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int
(result.total_run_time * 1000))} |
396 stats_trie = {} | 417 stats_trie = {} |
397 for name, value in stats.iteritems(): | 418 for name, value in stats.iteritems(): |
398 json_results_generator.add_path_to_trie(name, value, stats_trie) | 419 json_results_generator.add_path_to_trie(name, value, stats_trie) |
399 return stats_trie | 420 return stats_trie |
OLD | NEW |