Index: tools/testing/test_runner.py
|
diff --git a/tools/testing/test_runner.py b/tools/testing/test_runner.py
|
deleted file mode 100755
|
index 9b2a6e0e121b8b678f5c8413a1cc8a4a4b46cc4e..0000000000000000000000000000000000000000
|
--- a/tools/testing/test_runner.py
|
+++ /dev/null
|
@@ -1,430 +0,0 @@
|
-# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
|
-# for details. All rights reserved. Use of this source code is governed by a
|
-# BSD-style license that can be found in the LICENSE file.
|
-#
|
-"""Classes and methods for executing tasks for the test.py framework.
|
-
|
-This module includes:
|
- - Managing parallel execution of tests using threads
|
- - Windows and Unix specific code for spawning tasks and retrieving results
|
- - Evaluating the output of each test as pass/fail/crash/timeout
|
-"""
|
-
|
-import ctypes
|
-import os
|
-import Queue
|
-import signal
|
-import subprocess
|
-import sys
|
-import tempfile
|
-import threading
|
-import time
|
-import traceback
|
-
|
-import testing
|
-import utils
|
-
|
-
|
-class Error(Exception):
|
- pass
|
-
|
-
|
-class CommandOutput(object):
|
- """Represents the output of running a command."""
|
-
|
- def __init__(self, pid, exit_code, timed_out, stdout, stderr):
|
- self.pid = pid
|
- self.exit_code = exit_code
|
- self.timed_out = timed_out
|
- self.stdout = stdout
|
- self.stderr = stderr
|
- self.failed = None
|
-
|
-
|
-class TestOutput(object):
|
- """Represents the output of running a TestCase."""
|
-
|
- def __init__(self, test, command, output):
|
- """Represents the output of running a TestCase.
|
-
|
- Args:
|
- test: A TestCase instance.
|
- command: the command line that was run
|
- output: A CommandOutput instance.
|
- """
|
- self.test = test
|
- self.command = command
|
- self.output = output
|
-
|
- def UnexpectedOutput(self):
|
- """Compare the result of running the expected from the TestConfiguration.
|
-
|
- Returns:
|
- True if the test had an unexpected output.
|
- """
|
- return not self.GetOutcome() in self.test.outcomes
|
-
|
- def GetOutcome(self):
|
- """Returns one of testing.CRASH, testing.TIMEOUT, testing.FAIL, or
|
- testing.PASS."""
|
- if self.HasCrashed():
|
- return testing.CRASH
|
- if self.HasTimedOut():
|
- return testing.TIMEOUT
|
- if self.HasFailed():
|
- return testing.FAIL
|
- return testing.PASS
|
-
|
- def HasCrashed(self):
|
- """Returns True if the test should be considered testing.CRASH."""
|
- if utils.IsWindows():
|
- if self.output.exit_code == 3:
|
- # The VM uses std::abort to terminate on asserts.
|
- # std::abort terminates with exit code 3 on Windows.
|
- return True
|
- return (0x80000000 & self.output.exit_code
|
- and not 0x3FFFFF00 & self.output.exit_code)
|
- else:
|
- # Timed out tests will have exit_code -signal.SIGTERM.
|
- if self.output.timed_out:
|
- return False
|
- if self.output.exit_code == 253:
|
- # The Java dartc runners exit 253 in case of unhandled exceptions.
|
- return True
|
- return self.output.exit_code < 0
|
-
|
- def HasTimedOut(self):
|
- """Returns True if the test should be considered as testing.TIMEOUT."""
|
- return self.output.timed_out
|
-
|
- def HasFailed(self):
|
- """Returns True if the test should be considered as testing.FAIL."""
|
- execution_failed = self.test.DidFail(self.output)
|
- if self.test.IsNegative():
|
- return not execution_failed
|
- else:
|
- return execution_failed
|
-
|
-
|
-def Execute(args, context, timeout=None, cwd=None):
|
- """Executes the specified command.
|
-
|
- Args:
|
- args: sequence of the executable name + arguments.
|
- context: An instance of Context object with global settings for test.py.
|
- timeout: optional timeout to wait for results in seconds.
|
- cwd: optionally change to this working directory.
|
-
|
- Returns:
|
- An instance of CommandOutput with the collected results.
|
- """
|
- (fd_out, outname) = tempfile.mkstemp()
|
- (fd_err, errname) = tempfile.mkstemp()
|
- (process, exit_code, timed_out) = RunProcess(context, timeout, args=args,
|
- stdout=fd_out, stderr=fd_err,
|
- cwd=cwd)
|
- os.close(fd_out)
|
- os.close(fd_err)
|
- output = file(outname).read()
|
- errors = file(errname).read()
|
- utils.CheckedUnlink(outname)
|
- utils.CheckedUnlink(errname)
|
- result = CommandOutput(process.pid, exit_code, timed_out,
|
- output, errors)
|
- return result
|
-
|
-
|
-def KillProcessWithID(pid):
|
- """Stop a process (with SIGTERM on Unix)."""
|
- if utils.IsWindows():
|
- os.popen('taskkill /T /F /PID %d' % pid)
|
- else:
|
- os.kill(pid, signal.SIGTERM)
|
-
|
-
|
-MAX_SLEEP_TIME = 0.1
|
-INITIAL_SLEEP_TIME = 0.0001
|
-SLEEP_TIME_FACTOR = 1.25
|
-SEM_INVALID_VALUE = -1
|
-SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h
|
-
|
-
|
-def Win32SetErrorMode(mode):
|
- """Some weird Windows stuff you just have to do."""
|
- prev_error_mode = SEM_INVALID_VALUE
|
- try:
|
- prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode)
|
- except ImportError:
|
- pass
|
- return prev_error_mode
|
-
|
-
|
-def RunProcess(context, timeout, args, **rest):
|
- """Handles the OS specific details of running a task and saving results."""
|
- if context.verbose: print '#', ' '.join(args)
|
- popen_args = args
|
- prev_error_mode = SEM_INVALID_VALUE
|
- if utils.IsWindows():
|
- popen_args = '"' + subprocess.list2cmdline(args) + '"'
|
- if context.suppress_dialogs:
|
- # Try to change the error mode to avoid dialogs on fatal errors. Don't
|
- # touch any existing error mode flags by merging the existing error mode.
|
- # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
|
- error_mode = SEM_NOGPFAULTERRORBOX
|
- prev_error_mode = Win32SetErrorMode(error_mode)
|
- Win32SetErrorMode(error_mode | prev_error_mode)
|
- process = subprocess.Popen(shell=utils.IsWindows(),
|
- args=popen_args,
|
- **rest)
|
- if (utils.IsWindows() and context.suppress_dialogs
|
- and prev_error_mode != SEM_INVALID_VALUE):
|
- Win32SetErrorMode(prev_error_mode)
|
- # Compute the end time - if the process crosses this limit we
|
- # consider it timed out.
|
- if timeout is None: end_time = None
|
- else: end_time = time.time() + timeout
|
- timed_out = False
|
- # Repeatedly check the exit code from the process in a
|
- # loop and keep track of whether or not it times out.
|
- exit_code = None
|
- sleep_time = INITIAL_SLEEP_TIME
|
- while exit_code is None:
|
- if (not end_time is None) and (time.time() >= end_time):
|
- # Kill the process and wait for it to exit.
|
- KillProcessWithID(process.pid)
|
- # Drain the output pipe from the process to avoid deadlock
|
- process.communicate()
|
- exit_code = process.wait()
|
- timed_out = True
|
- else:
|
- exit_code = process.poll()
|
- time.sleep(sleep_time)
|
- sleep_time *= SLEEP_TIME_FACTOR
|
- if sleep_time > MAX_SLEEP_TIME:
|
- sleep_time = MAX_SLEEP_TIME
|
- return (process, exit_code, timed_out)
|
-
|
-
|
-class TestRunner(object):
|
- """Base class for runners."""
|
-
|
- def __init__(self, work_queue, tasks, progress):
|
- self.work_queue = work_queue
|
- self.tasks = tasks
|
- self.terminate = False
|
- self.progress = progress
|
- self.threads = []
|
- self.shutdown_lock = threading.Lock()
|
-
|
-
|
-class BatchRunner(TestRunner):
|
- """Implements communication with a set of subprocesses using threads."""
|
-
|
- def __init__(self, work_queue, tasks, progress, batch_cmd):
|
- super(BatchRunner, self).__init__(work_queue, tasks, progress)
|
- self.runners = {}
|
- self.last_activity = {}
|
- self.context = progress.context
|
-
|
- # Scale the number of tasks to the nubmer of CPUs on the machine
|
- # 1:1 is too much of an overload on many machines in batch mode,
|
- # so scale the ratio of threads to CPUs back. On Windows running
|
- # more than one task is not safe.
|
- if tasks == testing.USE_DEFAULT_CPUS:
|
- if utils.IsWindows():
|
- tasks = 1
|
- else:
|
- tasks = .75 * testing.HOST_CPUS
|
-
|
- # Start threads
|
- for i in xrange(tasks):
|
- thread = threading.Thread(target=self.RunThread, args=[batch_cmd, i])
|
- self.threads.append(thread)
|
- thread.daemon = True
|
- thread.start()
|
-
|
- def RunThread(self, batch_cmd, thread_number):
|
- """A thread started to feed a single TestRunner."""
|
- try:
|
- runner = None
|
- while not self.terminate and not self.work_queue.empty():
|
- runner = subprocess.Popen(batch_cmd,
|
- stdin=subprocess.PIPE,
|
- stderr=subprocess.STDOUT,
|
- stdout=subprocess.PIPE)
|
- self.runners[thread_number] = runner
|
- self.FeedTestRunner(runner, thread_number)
|
- if thread_number in self.last_activity:
|
- del self.last_activity[thread_number]
|
-
|
- # Cleanup
|
- self.EndRunner(runner)
|
-
|
- except:
|
- self.Shutdown()
|
- raise
|
- finally:
|
- if thread_number in self.last_activity:
|
- del self.last_activity[thread_number]
|
- if runner: self.EndRunner(runner)
|
-
|
- def EndRunner(self, runner):
|
- """Cleans up a single runner, killing the child if necessary."""
|
- with self.shutdown_lock:
|
- if runner:
|
- returncode = runner.poll()
|
- if returncode is None:
|
- runner.kill()
|
- for (found_runner, thread_number) in self.runners.items():
|
- if runner == found_runner:
|
- del self.runners[thread_number]
|
- break
|
- try:
|
- runner.communicate()
|
- except ValueError:
|
- pass
|
-
|
- def CheckForTimeouts(self):
|
- now = time.time()
|
- for (thread_number, start_time) in self.last_activity.items():
|
- if now - start_time > self.context.timeout:
|
- self.runners[thread_number].kill()
|
-
|
- def WaitForCompletion(self):
|
- """Wait for threads to finish, and monitor test runners for timeouts."""
|
- for t in self.threads:
|
- while True:
|
- self.CheckForTimeouts()
|
- t.join(timeout=5)
|
- if not t.isAlive():
|
- break
|
-
|
- def FeedTestRunner(self, runner, thread_number):
|
- """Feed commands to the fork'ed TestRunner through a Popen object."""
|
-
|
- last_case = {}
|
- last_buf = ''
|
-
|
- while not self.terminate:
|
- # Is the runner still alive?
|
- returninfo = runner.poll()
|
- if returninfo is not None:
|
- buf = last_buf + '\n' + runner.stdout.read()
|
- if last_case:
|
- self.RecordPassFail(last_case, buf, testing.CRASH)
|
- else:
|
- with self.progress.lock:
|
- print >>sys. stderr, ('%s: runner unexpectedly exited: %d'
|
- % (threading.currentThread().name,
|
- returninfo))
|
- print 'Crash Output: '
|
- print
|
- print buf
|
- return
|
-
|
- try:
|
- case = self.work_queue.get_nowait()
|
- with self.progress.lock:
|
- self.progress.AboutToRun(case.case)
|
-
|
- except Queue.Empty:
|
- return
|
- test_case = case.case
|
- cmd = ' '.join(test_case.GetCommand()[1:])
|
-
|
- try:
|
- print >>runner.stdin, cmd
|
- except IOError:
|
- with self.progress.lock:
|
- traceback.print_exc()
|
-
|
- # Child exited before starting the next command.
|
- buf = last_buf + '\n' + runner.stdout.read()
|
- self.RecordPassFail(last_case, buf, testing.CRASH)
|
-
|
- # We never got a chance to run this command - queue it back up.
|
- self.work_queue.put(case)
|
- return
|
-
|
- buf = ''
|
- self.last_activity[thread_number] = time.time()
|
- while not self.terminate:
|
- line = runner.stdout.readline()
|
- if self.terminate:
|
- break
|
- case.case.duration = time.time() - self.last_activity[thread_number]
|
- if not line:
|
- # EOF. Child has exited.
|
- if case.case.duration > self.context.timeout:
|
- with self.progress.lock:
|
- print 'Child timed out after %d seconds' % self.context.timeout
|
- self.RecordPassFail(case, buf, testing.TIMEOUT)
|
- elif buf:
|
- self.RecordPassFail(case, buf, testing.CRASH)
|
- return
|
-
|
- # Look for TestRunner batch status escape sequence. e.g.
|
- # >>> TEST PASS
|
- if line.startswith('>>> '):
|
- result = line.split()
|
- if result[1] == 'TEST':
|
- outcome = result[2].lower()
|
-
|
- # Read the rest of the output buffer (possible crash output)
|
- if outcome == testing.CRASH:
|
- buf += runner.stdout.read()
|
-
|
- self.RecordPassFail(case, buf, outcome)
|
-
|
- # Always handle crashes by restarting the runner.
|
- if outcome == testing.CRASH:
|
- return
|
- break
|
- elif result[1] == 'BATCH':
|
- pass
|
- else:
|
- print 'Unknown cmd from batch runner: %s' % line
|
- else:
|
- buf += line
|
-
|
- # If the process crashes before the next command is executed,
|
- # save info to report diagnostics.
|
- last_buf = buf
|
- last_case = case
|
-
|
- def RecordPassFail(self, case, stdout_buf, outcome):
|
- """An unexpected failure occurred."""
|
- if outcome == testing.PASS or outcome == testing.OKAY:
|
- exit_code = 0
|
- elif outcome == testing.CRASH:
|
- exit_code = -1
|
- elif outcome == testing.FAIL or outcome == testing.TIMEOUT:
|
- exit_code = 1
|
- else:
|
- assert False, 'Unexpected outcome: %s' % outcome
|
-
|
- cmd_output = CommandOutput(0, exit_code,
|
- outcome == testing.TIMEOUT, stdout_buf, '')
|
- test_output = TestOutput(case.case,
|
- case.case.GetCommand(),
|
- cmd_output)
|
- with self.progress.lock:
|
- if test_output.UnexpectedOutput():
|
- self.progress.failed.append(test_output)
|
- else:
|
- self.progress.succeeded += 1
|
- if outcome == testing.CRASH:
|
- self.progress.crashed += 1
|
- self.progress.remaining -= 1
|
- self.progress.HasRun(test_output)
|
-
|
- def Shutdown(self):
|
- """Kill all active runners."""
|
- print 'Shutting down remaining runners.'
|
- self.terminate = True
|
- for runner in self.runners.values():
|
- runner.kill()
|
- # Give threads a chance to exit gracefully
|
- time.sleep(2)
|
- for runner in self.runners.values():
|
- self.EndRunner(runner)
|
|