| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
| 2 # for details. All rights reserved. Use of this source code is governed by a | |
| 3 # BSD-style license that can be found in the LICENSE file. | |
| 4 # | |
| 5 """Classes and methods for executing tasks for the test.py framework. | |
| 6 | |
| 7 This module includes: | |
| 8 - Managing parallel execution of tests using threads | |
| 9 - Windows and Unix specific code for spawning tasks and retrieving results | |
| 10 - Evaluating the output of each test as pass/fail/crash/timeout | |
| 11 """ | |
| 12 | |
| 13 import ctypes | |
| 14 import os | |
| 15 import Queue | |
| 16 import signal | |
| 17 import subprocess | |
| 18 import sys | |
| 19 import tempfile | |
| 20 import threading | |
| 21 import time | |
| 22 import traceback | |
| 23 | |
| 24 import testing | |
| 25 import utils | |
| 26 | |
| 27 | |
| 28 class Error(Exception): | |
| 29 pass | |
| 30 | |
| 31 | |
| 32 class CommandOutput(object): | |
| 33 """Represents the output of running a command.""" | |
| 34 | |
| 35 def __init__(self, pid, exit_code, timed_out, stdout, stderr): | |
| 36 self.pid = pid | |
| 37 self.exit_code = exit_code | |
| 38 self.timed_out = timed_out | |
| 39 self.stdout = stdout | |
| 40 self.stderr = stderr | |
| 41 self.failed = None | |
| 42 | |
| 43 | |
| 44 class TestOutput(object): | |
| 45 """Represents the output of running a TestCase.""" | |
| 46 | |
| 47 def __init__(self, test, command, output): | |
| 48 """Represents the output of running a TestCase. | |
| 49 | |
| 50 Args: | |
| 51 test: A TestCase instance. | |
| 52 command: the command line that was run | |
| 53 output: A CommandOutput instance. | |
| 54 """ | |
| 55 self.test = test | |
| 56 self.command = command | |
| 57 self.output = output | |
| 58 | |
| 59 def UnexpectedOutput(self): | |
| 60 """Compare the result of running the expected from the TestConfiguration. | |
| 61 | |
| 62 Returns: | |
| 63 True if the test had an unexpected output. | |
| 64 """ | |
| 65 return not self.GetOutcome() in self.test.outcomes | |
| 66 | |
| 67 def GetOutcome(self): | |
| 68 """Returns one of testing.CRASH, testing.TIMEOUT, testing.FAIL, or | |
| 69 testing.PASS.""" | |
| 70 if self.HasCrashed(): | |
| 71 return testing.CRASH | |
| 72 if self.HasTimedOut(): | |
| 73 return testing.TIMEOUT | |
| 74 if self.HasFailed(): | |
| 75 return testing.FAIL | |
| 76 return testing.PASS | |
| 77 | |
| 78 def HasCrashed(self): | |
| 79 """Returns True if the test should be considered testing.CRASH.""" | |
| 80 if utils.IsWindows(): | |
| 81 if self.output.exit_code == 3: | |
| 82 # The VM uses std::abort to terminate on asserts. | |
| 83 # std::abort terminates with exit code 3 on Windows. | |
| 84 return True | |
| 85 return (0x80000000 & self.output.exit_code | |
| 86 and not 0x3FFFFF00 & self.output.exit_code) | |
| 87 else: | |
| 88 # Timed out tests will have exit_code -signal.SIGTERM. | |
| 89 if self.output.timed_out: | |
| 90 return False | |
| 91 if self.output.exit_code == 253: | |
| 92 # The Java dartc runners exit 253 in case of unhandled exceptions. | |
| 93 return True | |
| 94 return self.output.exit_code < 0 | |
| 95 | |
| 96 def HasTimedOut(self): | |
| 97 """Returns True if the test should be considered as testing.TIMEOUT.""" | |
| 98 return self.output.timed_out | |
| 99 | |
| 100 def HasFailed(self): | |
| 101 """Returns True if the test should be considered as testing.FAIL.""" | |
| 102 execution_failed = self.test.DidFail(self.output) | |
| 103 if self.test.IsNegative(): | |
| 104 return not execution_failed | |
| 105 else: | |
| 106 return execution_failed | |
| 107 | |
| 108 | |
| 109 def Execute(args, context, timeout=None, cwd=None): | |
| 110 """Executes the specified command. | |
| 111 | |
| 112 Args: | |
| 113 args: sequence of the executable name + arguments. | |
| 114 context: An instance of Context object with global settings for test.py. | |
| 115 timeout: optional timeout to wait for results in seconds. | |
| 116 cwd: optionally change to this working directory. | |
| 117 | |
| 118 Returns: | |
| 119 An instance of CommandOutput with the collected results. | |
| 120 """ | |
| 121 (fd_out, outname) = tempfile.mkstemp() | |
| 122 (fd_err, errname) = tempfile.mkstemp() | |
| 123 (process, exit_code, timed_out) = RunProcess(context, timeout, args=args, | |
| 124 stdout=fd_out, stderr=fd_err, | |
| 125 cwd=cwd) | |
| 126 os.close(fd_out) | |
| 127 os.close(fd_err) | |
| 128 output = file(outname).read() | |
| 129 errors = file(errname).read() | |
| 130 utils.CheckedUnlink(outname) | |
| 131 utils.CheckedUnlink(errname) | |
| 132 result = CommandOutput(process.pid, exit_code, timed_out, | |
| 133 output, errors) | |
| 134 return result | |
| 135 | |
| 136 | |
| 137 def KillProcessWithID(pid): | |
| 138 """Stop a process (with SIGTERM on Unix).""" | |
| 139 if utils.IsWindows(): | |
| 140 os.popen('taskkill /T /F /PID %d' % pid) | |
| 141 else: | |
| 142 os.kill(pid, signal.SIGTERM) | |
| 143 | |
| 144 | |
| 145 MAX_SLEEP_TIME = 0.1 | |
| 146 INITIAL_SLEEP_TIME = 0.0001 | |
| 147 SLEEP_TIME_FACTOR = 1.25 | |
| 148 SEM_INVALID_VALUE = -1 | |
| 149 SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h | |
| 150 | |
| 151 | |
| 152 def Win32SetErrorMode(mode): | |
| 153 """Some weird Windows stuff you just have to do.""" | |
| 154 prev_error_mode = SEM_INVALID_VALUE | |
| 155 try: | |
| 156 prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode) | |
| 157 except ImportError: | |
| 158 pass | |
| 159 return prev_error_mode | |
| 160 | |
| 161 | |
| 162 def RunProcess(context, timeout, args, **rest): | |
| 163 """Handles the OS specific details of running a task and saving results.""" | |
| 164 if context.verbose: print '#', ' '.join(args) | |
| 165 popen_args = args | |
| 166 prev_error_mode = SEM_INVALID_VALUE | |
| 167 if utils.IsWindows(): | |
| 168 popen_args = '"' + subprocess.list2cmdline(args) + '"' | |
| 169 if context.suppress_dialogs: | |
| 170 # Try to change the error mode to avoid dialogs on fatal errors. Don't | |
| 171 # touch any existing error mode flags by merging the existing error mode. | |
| 172 # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx. | |
| 173 error_mode = SEM_NOGPFAULTERRORBOX | |
| 174 prev_error_mode = Win32SetErrorMode(error_mode) | |
| 175 Win32SetErrorMode(error_mode | prev_error_mode) | |
| 176 process = subprocess.Popen(shell=utils.IsWindows(), | |
| 177 args=popen_args, | |
| 178 **rest) | |
| 179 if (utils.IsWindows() and context.suppress_dialogs | |
| 180 and prev_error_mode != SEM_INVALID_VALUE): | |
| 181 Win32SetErrorMode(prev_error_mode) | |
| 182 # Compute the end time - if the process crosses this limit we | |
| 183 # consider it timed out. | |
| 184 if timeout is None: end_time = None | |
| 185 else: end_time = time.time() + timeout | |
| 186 timed_out = False | |
| 187 # Repeatedly check the exit code from the process in a | |
| 188 # loop and keep track of whether or not it times out. | |
| 189 exit_code = None | |
| 190 sleep_time = INITIAL_SLEEP_TIME | |
| 191 while exit_code is None: | |
| 192 if (not end_time is None) and (time.time() >= end_time): | |
| 193 # Kill the process and wait for it to exit. | |
| 194 KillProcessWithID(process.pid) | |
| 195 # Drain the output pipe from the process to avoid deadlock | |
| 196 process.communicate() | |
| 197 exit_code = process.wait() | |
| 198 timed_out = True | |
| 199 else: | |
| 200 exit_code = process.poll() | |
| 201 time.sleep(sleep_time) | |
| 202 sleep_time *= SLEEP_TIME_FACTOR | |
| 203 if sleep_time > MAX_SLEEP_TIME: | |
| 204 sleep_time = MAX_SLEEP_TIME | |
| 205 return (process, exit_code, timed_out) | |
| 206 | |
| 207 | |
| 208 class TestRunner(object): | |
| 209 """Base class for runners.""" | |
| 210 | |
| 211 def __init__(self, work_queue, tasks, progress): | |
| 212 self.work_queue = work_queue | |
| 213 self.tasks = tasks | |
| 214 self.terminate = False | |
| 215 self.progress = progress | |
| 216 self.threads = [] | |
| 217 self.shutdown_lock = threading.Lock() | |
| 218 | |
| 219 | |
| 220 class BatchRunner(TestRunner): | |
| 221 """Implements communication with a set of subprocesses using threads.""" | |
| 222 | |
| 223 def __init__(self, work_queue, tasks, progress, batch_cmd): | |
| 224 super(BatchRunner, self).__init__(work_queue, tasks, progress) | |
| 225 self.runners = {} | |
| 226 self.last_activity = {} | |
| 227 self.context = progress.context | |
| 228 | |
| 229 # Scale the number of tasks to the nubmer of CPUs on the machine | |
| 230 # 1:1 is too much of an overload on many machines in batch mode, | |
| 231 # so scale the ratio of threads to CPUs back. On Windows running | |
| 232 # more than one task is not safe. | |
| 233 if tasks == testing.USE_DEFAULT_CPUS: | |
| 234 if utils.IsWindows(): | |
| 235 tasks = 1 | |
| 236 else: | |
| 237 tasks = .75 * testing.HOST_CPUS | |
| 238 | |
| 239 # Start threads | |
| 240 for i in xrange(tasks): | |
| 241 thread = threading.Thread(target=self.RunThread, args=[batch_cmd, i]) | |
| 242 self.threads.append(thread) | |
| 243 thread.daemon = True | |
| 244 thread.start() | |
| 245 | |
| 246 def RunThread(self, batch_cmd, thread_number): | |
| 247 """A thread started to feed a single TestRunner.""" | |
| 248 try: | |
| 249 runner = None | |
| 250 while not self.terminate and not self.work_queue.empty(): | |
| 251 runner = subprocess.Popen(batch_cmd, | |
| 252 stdin=subprocess.PIPE, | |
| 253 stderr=subprocess.STDOUT, | |
| 254 stdout=subprocess.PIPE) | |
| 255 self.runners[thread_number] = runner | |
| 256 self.FeedTestRunner(runner, thread_number) | |
| 257 if thread_number in self.last_activity: | |
| 258 del self.last_activity[thread_number] | |
| 259 | |
| 260 # Cleanup | |
| 261 self.EndRunner(runner) | |
| 262 | |
| 263 except: | |
| 264 self.Shutdown() | |
| 265 raise | |
| 266 finally: | |
| 267 if thread_number in self.last_activity: | |
| 268 del self.last_activity[thread_number] | |
| 269 if runner: self.EndRunner(runner) | |
| 270 | |
| 271 def EndRunner(self, runner): | |
| 272 """Cleans up a single runner, killing the child if necessary.""" | |
| 273 with self.shutdown_lock: | |
| 274 if runner: | |
| 275 returncode = runner.poll() | |
| 276 if returncode is None: | |
| 277 runner.kill() | |
| 278 for (found_runner, thread_number) in self.runners.items(): | |
| 279 if runner == found_runner: | |
| 280 del self.runners[thread_number] | |
| 281 break | |
| 282 try: | |
| 283 runner.communicate() | |
| 284 except ValueError: | |
| 285 pass | |
| 286 | |
| 287 def CheckForTimeouts(self): | |
| 288 now = time.time() | |
| 289 for (thread_number, start_time) in self.last_activity.items(): | |
| 290 if now - start_time > self.context.timeout: | |
| 291 self.runners[thread_number].kill() | |
| 292 | |
| 293 def WaitForCompletion(self): | |
| 294 """Wait for threads to finish, and monitor test runners for timeouts.""" | |
| 295 for t in self.threads: | |
| 296 while True: | |
| 297 self.CheckForTimeouts() | |
| 298 t.join(timeout=5) | |
| 299 if not t.isAlive(): | |
| 300 break | |
| 301 | |
| 302 def FeedTestRunner(self, runner, thread_number): | |
| 303 """Feed commands to the fork'ed TestRunner through a Popen object.""" | |
| 304 | |
| 305 last_case = {} | |
| 306 last_buf = '' | |
| 307 | |
| 308 while not self.terminate: | |
| 309 # Is the runner still alive? | |
| 310 returninfo = runner.poll() | |
| 311 if returninfo is not None: | |
| 312 buf = last_buf + '\n' + runner.stdout.read() | |
| 313 if last_case: | |
| 314 self.RecordPassFail(last_case, buf, testing.CRASH) | |
| 315 else: | |
| 316 with self.progress.lock: | |
| 317 print >>sys. stderr, ('%s: runner unexpectedly exited: %d' | |
| 318 % (threading.currentThread().name, | |
| 319 returninfo)) | |
| 320 print 'Crash Output: ' | |
| 321 print | |
| 322 print buf | |
| 323 return | |
| 324 | |
| 325 try: | |
| 326 case = self.work_queue.get_nowait() | |
| 327 with self.progress.lock: | |
| 328 self.progress.AboutToRun(case.case) | |
| 329 | |
| 330 except Queue.Empty: | |
| 331 return | |
| 332 test_case = case.case | |
| 333 cmd = ' '.join(test_case.GetCommand()[1:]) | |
| 334 | |
| 335 try: | |
| 336 print >>runner.stdin, cmd | |
| 337 except IOError: | |
| 338 with self.progress.lock: | |
| 339 traceback.print_exc() | |
| 340 | |
| 341 # Child exited before starting the next command. | |
| 342 buf = last_buf + '\n' + runner.stdout.read() | |
| 343 self.RecordPassFail(last_case, buf, testing.CRASH) | |
| 344 | |
| 345 # We never got a chance to run this command - queue it back up. | |
| 346 self.work_queue.put(case) | |
| 347 return | |
| 348 | |
| 349 buf = '' | |
| 350 self.last_activity[thread_number] = time.time() | |
| 351 while not self.terminate: | |
| 352 line = runner.stdout.readline() | |
| 353 if self.terminate: | |
| 354 break | |
| 355 case.case.duration = time.time() - self.last_activity[thread_number] | |
| 356 if not line: | |
| 357 # EOF. Child has exited. | |
| 358 if case.case.duration > self.context.timeout: | |
| 359 with self.progress.lock: | |
| 360 print 'Child timed out after %d seconds' % self.context.timeout | |
| 361 self.RecordPassFail(case, buf, testing.TIMEOUT) | |
| 362 elif buf: | |
| 363 self.RecordPassFail(case, buf, testing.CRASH) | |
| 364 return | |
| 365 | |
| 366 # Look for TestRunner batch status escape sequence. e.g. | |
| 367 # >>> TEST PASS | |
| 368 if line.startswith('>>> '): | |
| 369 result = line.split() | |
| 370 if result[1] == 'TEST': | |
| 371 outcome = result[2].lower() | |
| 372 | |
| 373 # Read the rest of the output buffer (possible crash output) | |
| 374 if outcome == testing.CRASH: | |
| 375 buf += runner.stdout.read() | |
| 376 | |
| 377 self.RecordPassFail(case, buf, outcome) | |
| 378 | |
| 379 # Always handle crashes by restarting the runner. | |
| 380 if outcome == testing.CRASH: | |
| 381 return | |
| 382 break | |
| 383 elif result[1] == 'BATCH': | |
| 384 pass | |
| 385 else: | |
| 386 print 'Unknown cmd from batch runner: %s' % line | |
| 387 else: | |
| 388 buf += line | |
| 389 | |
| 390 # If the process crashes before the next command is executed, | |
| 391 # save info to report diagnostics. | |
| 392 last_buf = buf | |
| 393 last_case = case | |
| 394 | |
| 395 def RecordPassFail(self, case, stdout_buf, outcome): | |
| 396 """An unexpected failure occurred.""" | |
| 397 if outcome == testing.PASS or outcome == testing.OKAY: | |
| 398 exit_code = 0 | |
| 399 elif outcome == testing.CRASH: | |
| 400 exit_code = -1 | |
| 401 elif outcome == testing.FAIL or outcome == testing.TIMEOUT: | |
| 402 exit_code = 1 | |
| 403 else: | |
| 404 assert False, 'Unexpected outcome: %s' % outcome | |
| 405 | |
| 406 cmd_output = CommandOutput(0, exit_code, | |
| 407 outcome == testing.TIMEOUT, stdout_buf, '') | |
| 408 test_output = TestOutput(case.case, | |
| 409 case.case.GetCommand(), | |
| 410 cmd_output) | |
| 411 with self.progress.lock: | |
| 412 if test_output.UnexpectedOutput(): | |
| 413 self.progress.failed.append(test_output) | |
| 414 else: | |
| 415 self.progress.succeeded += 1 | |
| 416 if outcome == testing.CRASH: | |
| 417 self.progress.crashed += 1 | |
| 418 self.progress.remaining -= 1 | |
| 419 self.progress.HasRun(test_output) | |
| 420 | |
| 421 def Shutdown(self): | |
| 422 """Kill all active runners.""" | |
| 423 print 'Shutting down remaining runners.' | |
| 424 self.terminate = True | |
| 425 for runner in self.runners.values(): | |
| 426 runner.kill() | |
| 427 # Give threads a chance to exit gracefully | |
| 428 time.sleep(2) | |
| 429 for runner in self.runners.values(): | |
| 430 self.EndRunner(runner) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 9360017:
Remove unused support files for old version of tools/test.py. (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart