Replace recipes_test.py with a new parallel expectation-based test runner.

scripts/slave/unittests/expect_tests/pipeline.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import Queue
+import glob
+import multiprocessing
+import re
+import signal
+
+from .types import Test, UnknownError, TestError, Result, ResultStageAbort
+
+
+def GenLoopProcess(gen, test_queue, result_queue, opts, kill_switch, cover_ctx):
+  """Generate `Test`'s from |gen|, and feed them into |test_queue|.
+
+  Non-Test instances will be translated into `UnknownError` objects.
+
+  On completion, feed |opts.jobs| None objects into |test_queue|.
+
+  @param gen: generator yielding Test() instances.
+  @type test_queue: multiprocessing.Queue()
+  @type result_queue: multiprocessing.Queue()
+  @type opts: argparse.Namespace
+  @type kill_switch: multiprocessing.Event()
+  @type match_globs: [str]
+  @type cover_ctx: cover.CoverageContext().create_subprocess_context()
+  @type handler: types.Handler
+  """
+  matcher = re.compile(
+      '^%s$' % '|'.join('(?:%s)' % glob.fnmatch.translate(g)
+                        for g in opts.test_glob if g[0] != '-'))
+  if matcher.pattern == '^$':
+    matcher = re.compile('^.*$')
+
+  neg_matcher = re.compile(
+      '^%s$' % '|'.join('(?:%s)' % glob.fnmatch.translate(g[1:])
+                        for g in opts.test_glob if g[0] == '-'))
+
+  def generate_tests():
+    try:
+      for test in gen():
+        if kill_switch.is_set():
+          break
+
+        if not isinstance(test, Test):
+          result_queue.put_nowait(
+              UnknownError(
+                  'Got non-Test isinstance from generator: %r' % test))
+          continue
+
+        if not neg_matcher.match(test.name) and matcher.match(test.name):
+          yield test
+    except KeyboardInterrupt:
+      pass
+    finally:
+      for _ in xrange(opts.jobs):
+        test_queue.put_nowait(None)
+
+
+  next_stage = (result_queue if opts.handler.SKIP_RUNLOOP else test_queue)
+  with cover_ctx:
+    opts.handler.gen_stage_loop(opts, generate_tests(), next_stage.put_nowait,
+                                result_queue.put_nowait)
+
+
+def RunLoopProcess(test_queue, result_queue, opts, kill_switch, cover_ctx):
+  """Consume `Test` instances from |test_queue|, run them, and yield the results
+  into opts.run_stage_loop().
+
+  Generates coverage data as a side-effect.
+  @type test_queue: multiprocessing.Queue()
+  @type result_queue: multiprocessing.Queue()
+  @type opts: argparse.Namespace
+  @type kill_switch: multiprocessing.Event()
+  @type cover_ctx: cover.CoverageContext().create_subprocess_context()
+  """
+  def generate_tests_results():
+    try:
+      while not kill_switch.is_set():
+        try:
+          test = test_queue.get(timeout=0.1)
+          if test is None:
+            break
+        except Queue.Empty:
+          continue
+
+        try:
+          result = test.run()
+          if not isinstance(result, Result):
+            result_queue.put_nowait(
+                TestError(test, 'Got non-Result instance from test: %r'
+                          % result))
+            continue
+
+          yield test, result
+        except Exception as e:
+          # TODO(iannucci): include stacktrace
+          result_queue.put_nowait(TestError(test, str(e)))
+    except KeyboardInterrupt:
+      pass
+
+  with cover_ctx:
+    opts.handler.run_stage_loop(opts, generate_tests_results(),
+                                result_queue.put_nowait)
+
+
+def ResultLoop(test_gen, cover_ctx, opts):
+  kill_switch = multiprocessing.Event()
+  def handle_killswitch(*_):
+    kill_switch.set()
+    signal.signal(signal.SIGINT, signal.SIG_DFL)

[Vadim Sh., 2014/04/02 22:37:59]

why setting to DFL? To die on double  Ctrl+C?

[iannucci, 2014/04/03 00:03:49]

Yeah, added comment.

+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+  signal.signal(signal.SIGINT, handle_killswitch)
+  signal.signal(signal.SIGTERM, handle_killswitch)
+
+  test_queue = multiprocessing.Queue()
+  result_queue = multiprocessing.Queue()
+
+  test_gen_args = (
+      test_gen, test_queue, result_queue, opts, kill_switch, cover_ctx)
+
+  procs = []
+  if opts.handler.SKIP_RUNLOOP:
+    GenLoopProcess(*test_gen_args)
+    class DeadProc(object):
+      """It's alive exactly once."""

[Vadim Sh., 2014/04/02 22:37:59]

ugh...

[iannucci, 2014/04/03 00:03:49]

Otherwise the generate_objects loop was racy :(. There's probably a refactor,
but I can't think of it at the moment.

+      alive = True
+      def is_alive(self):
+        ret = self.alive
+        self.alive = False
+        return ret
+    # This is so we can write generate_objects() in a non-racy way
+    procs = [DeadProc()]
+  else:
+    procs = [multiprocessing.Process(
+        target=GenLoopProcess, args=test_gen_args)]
+
+    procs += [
+        multiprocessing.Process(
+            target=RunLoopProcess, args=(
+                test_queue, result_queue, opts, kill_switch, cover_ctx))
+        for _ in xrange(opts.jobs)
+    ]
+
+    for p in procs:
+      p.daemon = True
+      p.start()
+
+  error = False
+  try:
+    def generate_objects():
+      while not kill_switch.is_set():
+        if not any(p.is_alive() for p in procs):
+          break
+
+        while not kill_switch.is_set():
+          try:
+            yield result_queue.get(timeout=0.1)
+          except Queue.Empty:
+            break
+      if kill_switch.is_set():
+        raise ResultStageAbort()
+    error = opts.handler.result_stage_loop(opts, generate_objects())
+  except ResultStageAbort:
+    pass
+
+  if not kill_switch.is_set() and not result_queue.empty():
+    error = True
+
+  return error, kill_switch.is_set()