[telemetry] test_runner and run_benchmarks

tools/telemetry/telemetry/test_runner.py

Index: tools/telemetry/telemetry/test_runner.py
diff --git a/tools/telemetry/telemetry/test_runner.py b/tools/telemetry/telemetry/test_runner.py
new file mode 100644
index 0000000000000000000000000000000000000000..57bf2c982acb42f700ec597d363b77ec55f2ab72
--- /dev/null
+++ b/tools/telemetry/telemetry/test_runner.py
@@ -0,0 +1,147 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Parses the command line, discovers the appropriate tests, and runs them.
+
+Handles test configuration, but all the logic for
+actually running the test is in PageRunner."""

[tonyg, 2013/06/19 16:40:15]

Recommend line breaks around the docstring

[dtu, 2013/06/19 21:22:58]

Done.

+import copy
+import inspect
+import optparse
+import os
+import sys
+
+from telemetry import test
+from telemetry.core import browser_options
+from telemetry.core import discover
+
+
+class Command(object):
+  usage = ''
+
+  @property
+  def name(self):
+    return self.__class__.__name__.lower()
+
+  @property
+  def description(self):
+    return self.__doc__
+
+  def CreateParser(self):
+    return optparse.OptionParser('%%prog %s %s' % (self.name, self.usage))
+
+  def ValidateCommandLine(self, parser, args, options):
+    pass
+
+  def Run(self, args, options):
+    raise NotImplementedError()
+
+
+class Help(Command):
+  """Display help information"""
+
+  def Run(self, args, options):
+    print >> sys.stderr, ('usage: %s <command> [<args>]' % _GetScriptName())
+    print >> sys.stderr, 'Available commands are:'
+    for command in COMMANDS:
+      print >> sys.stderr, '  %-10s %s' % (command.name, command.description)
+    return 0
+
+
+class List(Command):
+  """Lists the available tests"""
+

[nduca, 2013/06/19 05:46:53]

lets get a --json version of this in here quick so that people aren't tempted to
parse it...

[dtu, 2013/06/19 21:22:58]

Done.

+  def Run(self, args, options):
+    print >> sys.stderr, 'Available tests are:'
+    for test_name, test_class in sorted(_GetTests().items()):
+      print >> sys.stderr, '  %-20s %s' % (test_name,
+          test_class.__doc__.splitlines()[0])
+    return 0
+
+
+class Run(Command):
+  """Run one or more tests"""
+
+  usage = '[test_names ...] [<args>]'
+
+  def CreateParser(self):
+    options = browser_options.BrowserOptions()
+    parser = options.CreateParser('%%prog %s %s' % (self.name, self.usage))
+    return parser
+
+  def ValidateCommandLine(self, parser, test_names, options):
+    for test_name in test_names:
+      if test_name not in _GetTests():
+        parser.error('No test named "%s"' % test_name)
+
+  def Run(self, test_names, options):

[nduca, 2013/06/19 05:46:53]

bit awk that you have test_names here instead of args. I had to read the lower
code to realize where was just the args...

also, optparse is always options, args = parser.parse_args() --- eg options
comes first. Maybe swap order throughout to be normative? Minor nit.

[dtu, 2013/06/19 06:07:10]

Ugh, okay. It was like this because positional args always come before keyword
args.

[dtu, 2013/06/19 21:22:58]

Done.

+    if not test_names:

[nduca, 2013/06/19 05:46:53]

maybe we should parser.error in this case. i think running all benchmarks is
possibly more than someone might intend...

[dtu, 2013/06/19 21:22:58]

Done.

+      test_names = [test_name for test_name, test_class
+                    in _GetTests().iteritems() if test_class.enabled]
+
+    total_failures = 0
+    for test_name in test_names:
+      test_failures = _GetTests()[test_name]().Run(copy.copy(options))
+      total_failures += test_failures
+
+    return min(255, total_failures)
+
+
+COMMANDS = [cls() for _, cls in inspect.getmembers(sys.modules[__name__])
+            if inspect.isclass(cls)
+            and cls is not Command and issubclass(cls, Command)]
+
+
+def _GetScriptName():
+  return os.path.basename(sys.argv[0])
+
+
+def _GetBaseDir():
+  main_module = sys.modules['__main__']
+  if hasattr(main_module, '__file__'):
+    return os.path.dirname(main_module.__file__)
+  else:
+    return os.getcwd()
+
+
+def _GetTests():
+  # Lazy load and cache results.
+  if not hasattr(_GetTests, 'tests'):
+    base_dir = _GetBaseDir()
+    _GetTests.tests = discover.DiscoverClasses(base_dir, base_dir, test.Test,
+                                               index_by_class_name=True)
+  return _GetTests.tests
+
+
+def Main():
+  # Get the command name from the command line.
+  if len(sys.argv) > 1 and sys.argv[1] == '--help':
+    sys.argv[1] = 'help'
+
+  command_name = 'run'
+  for arg in sys.argv[1:]:
+    if not arg.startswith('-'):
+      command_name = arg
+      break
+
+  # Validate and interpret the command name.
+  commands = [command for command in COMMANDS
+              if command.name.startswith(command_name)]
+  if len(commands) > 1:
+    print >> sys.stderr, ('"%s" is not a %s command. Did you mean one of these?'
+                          % (command_name, _GetScriptName()))
+    for command in commands:
+      print >> sys.stderr, '  %-10s %s' % (command.name, command.description)
+    return 1
+  if commands:
+    command = commands[0]
+  else:
+    command = Run()
+
+  # Parse and run the command.
+  parser = command.CreateParser()
+  options, args = parser.parse_args()
+  if commands:
+    args = args[1:]
+  command.ValidateCommandLine(parser, args, options)
+  return command.Run(args, options)