Major refactoring for run_perf_tests.py.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import getpass
 import math
 try:
@@ skipping 13 matching lines @@
 import time
 import traceback
 
 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__)))))
 sys.path.append(TOOLS_PATH)
 import utils
 
 """This script runs to track performance and size progress of
 different svn revisions. It tests to see if there a newer version of the code on
 the server, and will sync and run the performance tests if so."""
-
-DART_INSTALL_LOCATION = abspath(os.path.join(dirname(abspath(__file__)),
-                                             '..', '..', '..'))
-_suffix = ''
-if platform.system() == 'Windows':
-  _suffix = '.exe'
-DART_VM = os.path.join(DART_INSTALL_LOCATION,
-                       utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'),
-                       'dart-sdk',
-                       'bin',
-                       'dart' + _suffix)
-DART_COMPILER = os.path.join(DART_INSTALL_LOCATION,
-                             utils.GetBuildRoot(utils.GuessOS(),
-                                                'release', 'ia32'),
-                             'dart-sdk',
-                             'bin',
-                             'frogc')
-
-GEO_MEAN = 'Geo-Mean'
-COMMAND_LINE = 'commandline'
-JS = 'js'
-FROG = 'frog'
-JS_AND_FROG = [JS, FROG]
-COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black']
-GRAPH_OUT_DIR = 'graphs'
-
-BROWSER_PERF = 'browser-perf'
-TIME_SIZE = 'time-size'
-CL_PERF = 'cl-perf'
-# TODO(vsm): Merge these?
-DROMAEO = 'dromaeo'
-DROMAEO_SIZE = 'dromaeo-size'
-
-SLEEP_TIME = 200
-VERBOSE = False
-HAS_SHELL = False
-if platform.system() == 'Windows':
-  # On Windows, shell must be true to get the correct environment variables.
-  HAS_SHELL = True
-
-"""First, some utility methods."""
-
-def run_cmd(cmd_list, outfile=None, append=False, std_in=''):
-  """Run the specified command and print out any output to stdout.
-
-  Args:
-    cmd_list: a list of strings that make up the command to run
-    outfile: a string indicating the name of the file that we should write
-       stdout to
-    append: True if we want to append to the file instead of overwriting it"""
-  if VERBOSE:
-    print ' '.join(cmd_list)
-  out = subprocess.PIPE
-  if outfile:
-    mode = 'w'
-    if append:
-      mode = 'a'
-    out = open(outfile, mode)
-    if append:
-      # Annoying Windows "feature" -- append doesn't actually append unless you
-      # explicitly go to the end of the file.
-      # http://mail.python.org/pipermail/python-list/2009-October/1221859.html
-      out.seek(0, os.SEEK_END)
-  p = subprocess.Popen(cmd_list, stdout = out, stderr=subprocess.PIPE,
-      stdin=subprocess.PIPE, shell=HAS_SHELL)
-  output, not_used = p.communicate(std_in);
-  if output:
-    print output
-  return output
-
-def time_cmd(cmd):
-  """Determine the amount of (real) time it takes to execute a given command."""
-  start = time.time()
-  run_cmd(cmd)
-  return time.time() - start
-
-def sync_and_build():
-  """Make sure we have the latest version of of the repo, and build it. We
-  begin and end standing in DART_INSTALL_LOCATION.
-
-  Returns:
-    err_code = 1 if there was a problem building."""
-  os.chdir(DART_INSTALL_LOCATION)
-  #Revert our newly built minfrog to prevent conflicts when we update
-  run_cmd(['svn', 'revert',  os.path.join(os.getcwd(), 'frog', 'minfrog')])
-
-  run_cmd(['gclient', 'sync'])
-
-  # On Windows, the output directory is marked as "Read Only," which causes an
-  # error to be thrown when we use shutil.rmtree. This helper function changes
-  # the permissions so we can still delete the directory.
-  def on_rm_error(func, path, exc_info):
-    if os.path.exists(path):
-      os.chmod(path, stat.S_IWRITE)
-      os.unlink(path)
-  # TODO(efortuna): building the sdk locally is a band-aid until all build
-  # platform SDKs are hosted in Google storage. Pull from https://sandbox.
-  # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk
-  # eventually.
-  # TODO(efortuna): Currently always building ia32 architecture because we don't
-  # have test statistics for what's passing on x64. Eliminate arch specification
-  # when we have tests running on x64, too.
-  shutil.rmtree(os.path.join(os.getcwd(),
-      utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32')),
-      onerror=on_rm_error)
-  lines = run_cmd([os.path.join('.', 'tools', 'build.py'), '-m', 'release',
-      '--arch=ia32', 'create_sdk'])
-  lines = run_cmd([os.path.join('.', 'tools', 'build.py'), '-m', 'release',
-      '--arch=ia32', 'dart2js']) #Built only for the v8 target for CL tests.
-
-  for line in lines:
-    if 'BUILD FAILED' in lines:
-      # Someone checked in a broken build! Just stop trying to make it work
-      # and wait to try again.
-      print 'Broken Build'
-      return 1
-  return 0
-
-def ensure_output_directory(dir_name):
-  """Test that the listed directory name exists, and if not, create one for
-  our output to be placed.
-
-  Args:
-    dir_name: the directory we will create if it does not exist."""
-  dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
-      'perf_testing', dir_name)
-  if not os.path.exists(dir_path):
-    os.mkdir(dir_path)
-    print 'Creating output directory ', dir_path
-
-def has_new_code():
-  """Tests if there are any newer versions of files on the server."""
-  os.chdir(DART_INSTALL_LOCATION)
-  # Pass 'p' in if we have a new certificate for the svn server, we want to
-  # (p)ermanently accept it.
-  results = run_cmd(['svn', 'st', '-u'], std_in='p')
-  for line in results:
-    if '*' in line:
-      return True
-  return False
-
-# TODO(vsm): Add Dartium.
-def get_browsers():
-  browsers = ['ff', 'chrome']
-  if platform.system() == 'Darwin':
-    browsers += ['safari']
-  if platform.system() == 'Windows':
-    browsers += ['ie']
-  return browsers
-
-# TODO(vsm): Factor benchmark specific code to a better location.
-def get_standalone_benchmarks():
-  return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees',
-  'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute',
-  'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers',
-  'TreeSort']
-
-def get_os_directory():
-  """Specifies the name of the directory for the testing build of dart, which
-  has yet a different naming convention from utils.getBuildRoot(...)."""
-  if platform.system() == 'Windows':
-    return 'windows'
-  elif platform.system() == 'Darwin':
-    return 'macos'
-  else:
-    return 'linux'
-
-def upload_to_app_engine(suite_names):
-  """Upload our results to our appengine server.
-  Arguments:
-    suite_names: Directories to upload data from (should match suite names)
-  """
-  # TODO(efortuna): This is the most basic way to get the data up
-  # for others to view. Revisit this once we're serving nicer graphs (Google
-  # Chart Tools) and from multiple perfbots and once we're in a position to
-  # organize the data in a useful manner(!!).
-  os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
-      'perf_testing'))
-  for data in suite_names:
-    path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS())
-    shutil.rmtree(path, ignore_errors=True)
-    os.makedirs(path)
-    files = []
-    # Copy the 1000 most recent trace files to be uploaded.
-    for f in os.listdir(data):
-      files += [(os.path.getmtime(os.path.join(data, f)), f)]
-    files.sort()
-    for f in files[-1000:]:
-      shutil.copyfile(os.path.join(data, f[1]),
-          os.path.join(path, f[1]+'.txt'))
-  # Generate directory listing.
-  for data in suite_names:
-    path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS())
-    out = open(os.path.join('appengine', 'static',
-        '%s-%s.html' % (data, utils.GuessOS())), 'w')
-    out.write('<html>\n  <body>\n    <ul>\n')
-    for f in os.listdir(path):
-      if not f.startswith('.'):
-        out.write('      <li><a href=data' + \
-            '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \
-            {'data': data, 'os': utils.GuessOS(), 'file': f})
-    out.write('    </ul>\n  </body>\n</html>')
-    out.close()
-
-  shutil.rmtree(os.path.join('appengine', 'static', 'graphs'),
-      ignore_errors=True)
-  shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs'))
-  shutil.copyfile('index.html', os.path.join('appengine', 'static',
-      'index.html'))
-  shutil.copyfile('dromaeo.html', os.path.join('appengine', 'static',
-      'dromaeo.html'))
-  shutil.copyfile('data.html', os.path.join('appengine', 'static',
-      'data.html'))
-  run_cmd([os.path.join('..', '..', '..', 'third_party',
-      'appengine-python', 'appcfg.py'), '--oauth2', 'update',
-      'appengine/'])
-
-
 class TestRunner(object):
+  DART_INSTALL_LOCATION = abspath(os.path.join(dirname(abspath(__file__)),

[vsm, 2012/04/09 20:19:58]

I'd hoist this up with TOOLS_PATH.

[Emily Fortuna, 2012/04/09 21:15:03]

Done.

+                                               '..', '..', '..'))
+  def __init__(self):
+    self.verbose = False
+    self.has_shell = False
+    if platform.system() == 'Windows':
+      # On Windows, shell must be true to get the correct environment variables.
+      self.has_shell = True
+
+  def run_cmd(self, cmd_list, outfile=None, append=False, std_in=''):
+    """Run the specified command and print out any output to stdout.
+
+    Args:
+      cmd_list: a list of strings that make up the command to run
+      outfile: a string indicating the name of the file that we should write
+         stdout to
+      append: True if we want to append to the file instead of overwriting it
+      std_in: a string that should be written to the process executing to
+         interact with it (if needed)"""
+    if self.verbose:
+      print ' '.join(cmd_list)
+    out = subprocess.PIPE
+    if outfile:
+      mode = 'w'
+      if append:
+        mode = 'a'
+      out = open(outfile, mode)
+      if append:
+        # Annoying Windows "feature" -- append doesn't actually append unless
+        # you explicitly go to the end of the file.
+        # http://mail.python.org/pipermail/python-list/2009-October/1221859.html
+        out.seek(0, os.SEEK_END)
+    p = subprocess.Popen(cmd_list, stdout = out, stderr=subprocess.PIPE,
+                         stdin=subprocess.PIPE, shell=self.has_shell)
+    output, not_used = p.communicate(std_in);

[vsm, 2012/04/09 20:19:58]

Not sure how standard this is, but I like:
  output, _ = ....
when I don't care about a value in the tuple.  Up to you.

[Emily Fortuna, 2012/04/09 21:15:03]

Done.

+    if output:
+      print output

[vsm, 2012/04/09 20:19:58]

Perhaps print only if self.verbose?

[Emily Fortuna, 2012/04/09 21:15:03]

This is how we get all of our output in our trace files, and error messages if
something goes terribly wrong. I think we want to keep this around pretty much
always.

[vsm, 2012/04/09 21:50:18]

Isn't trace file output going directly to the "out" file defined above?

But, I think it's fine to keep this as is if you prefer.

On 2012/04/09 21:15:03, Emily Fortuna wrote:

[Emily Fortuna, 2012/04/09 22:21:45]

Yes, you're right. This is one of those cases where I like to actually see that
the script is actually making progress instead of hanging.

+    return output
+
+  def time_cmd(self, cmd):
+    """Determine the amount of (real) time it takes to execute a given 
+    command."""
+    start = time.time()
+    self.run_cmd(cmd)
+    return time.time() - start
+
+  @staticmethod
+  def get_build_targets(suites):
+    """Loop through a set of tests that we want to run and find the build
+    targets that are necessary.
+    
+    Args:
+      suites: The test suites that we wish to run."""
+    build_targets = set()
+    for test in suites:
+      if test.build_targets is not None:
+        for target in test.build_targets:
+          build_targets.add(target)
+    return build_targets
+
+  def sync_and_build(self, suites):
+    """Make sure we have the latest version of of the repo, and build it. We
+    begin and end standing in DART_INSTALL_LOCATION.
+
+    Args:
+      suites: The set of suites that we wish to build.
+
+    Returns:
+      err_code = 1 if there was a problem building."""
+    os.chdir(TestRunner.DART_INSTALL_LOCATION)
+
+    self.run_cmd(['gclient', 'sync'])
+
+    # On Windows, the output directory is marked as "Read Only," which causes an
+    # error to be thrown when we use shutil.rmtree. This helper function changes
+    # the permissions so we can still delete the directory.
+    def on_rm_error(func, path, exc_info):
+      if os.path.exists(path):
+        os.chmod(path, stat.S_IWRITE)
+        os.unlink(path)
+    # TODO(efortuna): building the sdk locally is a band-aid until all build
+    # platform SDKs are hosted in Google storage. Pull from https://sandbox.
+    # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk
+    # eventually.
+    # TODO(efortuna): Currently always building ia32 architecture because we 
+    # don't have test statistics for what's passing on x64. Eliminate arch 
+    # specification when we have tests running on x64, too.
+    shutil.rmtree(os.path.join(os.getcwd(),
+                  utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32')),
+                  onerror=on_rm_error)
+
+    for target in TestRunner.get_build_targets(suites):
+      lines = self.run_cmd([os.path.join('.', 'tools', 'build.py'), '-m', 
+                            'release', '--arch=ia32', target])
+
+      for line in lines:
+        if 'BUILD FAILED' in lines:
+          # Someone checked in a broken build! Stop trying to make it work
+          # and wait to try again.
+          print 'Broken Build'
+          return 1
+    return 0
+
+  def ensure_output_directory(self, dir_name):
+    """Test that the listed directory name exists, and if not, create one for
+    our output to be placed.
+
+    Args:
+      dir_name: the directory we will create if it does not exist."""
+    dir_path = os.path.join(TestRunner.DART_INSTALL_LOCATION, 'tools', 
+                            'testing', 'perf_testing', dir_name)
+    if not os.path.exists(dir_path):
+      os.mkdir(dir_path)
+      print 'Creating output directory ', dir_path
+
+  def has_new_code(self):
+    """Tests if there are any newer versions of files on the server."""
+    os.chdir(TestRunner.DART_INSTALL_LOCATION)
+    # Pass 'p' in if we have a new certificate for the svn server, we want to
+    # (p)ermanently accept it.
+    results = self.run_cmd(['svn', 'st', '-u'], std_in='p')
+    for line in results:
+      if '*' in line:
+        return True
+    return False
+    
+  def get_os_directory(self):
+    """Specifies the name of the directory for the testing build of dart, which
+    has yet a different naming convention from utils.getBuildRoot(...)."""
+    if platform.system() == 'Windows':
+      return 'windows'
+    elif platform.system() == 'Darwin':
+      return 'macos'
+    else:
+      return 'linux'
+
+  def upload_to_app_engine(self, suite_names):
+    """Upload our results to our appengine server.
+    Arguments:
+      suite_names: Directories to upload data from (should match directory 
+        names)."""
+    # TODO(efortuna): This is the most basic way to get the data up
+    # for others to view. Revisit this once we're serving nicer graphs (Google
+    # Chart Tools) and from multiple perfbots and once we're in a position to
+    # organize the data in a useful manner(!!).
+    os.chdir(os.path.join(TestRunner.DART_INSTALL_LOCATION, 'tools', 'testing',
+                          'perf_testing'))
+    for data in suite_names:
+      path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS())
+      shutil.rmtree(path, ignore_errors=True)
+      os.makedirs(path)
+      files = []
+      # Copy the 1000 most recent trace files to be uploaded.
+      for f in os.listdir(data):
+        files += [(os.path.getmtime(os.path.join(data, f)), f)]
+      files.sort()
+      for f in files[-1000:]:
+        shutil.copyfile(os.path.join(data, f[1]), 
+                        os.path.join(path, f[1]+'.txt'))
+    # Generate directory listing.
+    for data in suite_names:
+      path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS())
+      out = open(os.path.join('appengine', 'static',
+                 '%s-%s.html' % (data, utils.GuessOS())), 'w')
+      out.write('<html>\n  <body>\n    <ul>\n')
+      for f in os.listdir(path):
+        if not f.startswith('.'):
+          out.write('      <li><a href=data' + \
+                    '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \
+                    {'data': data, 'os': utils.GuessOS(), 'file': f})
+      out.write('    </ul>\n  </body>\n</html>')
+      out.close()
+
+    shutil.rmtree(os.path.join('appengine', 'static', 'graphs'),
+                  ignore_errors=True)
+    shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs'))
+    shutil.copyfile('index.html', os.path.join('appengine', 'static',
+                    'index.html'))
+    shutil.copyfile('dromaeo.html', os.path.join('appengine', 'static',
+                    'dromaeo.html'))
+    shutil.copyfile('data.html', os.path.join('appengine', 'static',
+                    'data.html'))
+    self.run_cmd([os.path.join('..', '..', '..', 'third_party',
+                               'appengine-python', 'appcfg.py'), '--oauth2', 
+                               'update', 'appengine/'])
+
+  def parse_args(self):
+    parser = optparse.OptionParser()
+    parser.add_option('--suites', '-s', dest='suites', help='Run the specified '
+                      'comma-separated test suites from set: %s' % \
+                      ','.join(TestBuilder.available_suite_names()), 
+                      action='store', default=None)
+    parser.add_option('--forever', '-f', dest='continuous', help='Run this scri'
+                      'pt forever, always checking for the next svn checkin',
+                      action='store_true', default=False)
+    parser.add_option('--graph-only', '-g', dest='graph_only', default=False,
+                      help='Do not run tests, only regenerate graphs', 
+                      action='store_true')
+    parser.add_option('--nobuild', '-n', dest='no_build', action='store_true',
+                      help='Do not sync with the repository and do not '
+                      'rebuild.', default=False)
+    parser.add_option('--upload', '-u', dest='upload', help='Upload data to '
+                      'app engine (will require authentication).', 
+                      action='store_true', default=False)
+    parser.add_option('--verbose', '-v', dest='verbose', help='Print extra '
+                      'debug output', action='store_true', default=False)
+
+    args, ignored = parser.parse_args()
+
+    if not args.suites:
+      suites = TestBuilder.available_suite_names()
+    else:
+      suites = []
+      suitelist = args.suites.split(',')
+      for name in suitelist:
+        if name in TestBuilder.available_suite_names():
+          suites.append(name)
+        else:
+          print ('Error: Invalid suite %s not in ' % name) + \
+              '%s' % ','.join(TestBuilder.available_suite_names())
+          sys.exit(1)
+    return (suites, args.continuous, args.verbose, args.no_build,
+            args.graph_only, args.upload)
+
+  def run_test_sequence(self, suite_names, no_build, graph_only, upload):
+    """Run the set of commands to (possibly) build, run, and graph the results
+    of our tests.
+
+    Args:
+      suite_names: The "display name" the user enters to specify which
+          benchmark(s) to run.
+      no_build: True if we should not check the repository and build the latest
+          version.
+      graph_only: True if we should not run the tests, just (re)generate graphs.
+      upload: True if we should upload our results to appengine."""
+    suites = []
+    for name in suite_names:
+      suites += [TestBuilder().make_test(name, self)]
+
+    if not no_build and self.sync_and_build(suites) == 1:
+      return # The build is broken.
+
+    for test in suites:
+      test.run(graph_only)
+
+    if upload:
+      self.upload_to_app_engine(TestBuilder.available_site_names())
+
+
+class Test(object):
   """The base class to provide shared code for different tests we will run and
-  graph."""
+  graph. At a high level, each test has three visitors (the tester, the
+  file_processor, and the grapher) that perform operations on the test 
+  object."""
 
   def __init__(self, result_folder_name, platform_list, variants,
-               values_list):
+               values_list, test_runner, tester, file_processor, grapher,
+               extra_metrics=['Geo-Mean'], build_targets=['create_sdk']):
     """Args:
-         result_folder_name the name of the folder where a tracefile of
-         performance results will be stored.
-         platform_list a list containing the platform(s) that our data has been
-            run on. (command line, firefox, chrome, etc)
-         variants a list specifying whether we hold data about Frog
-            generated code, plain JS code (js), or a combination of both.
-         values_list a list containing the type of data we will be graphing
-            (benchmarks, percentage passing, etc)"""
+         result_folder_name: The name of the folder where a tracefile of
+             performance results will be stored.
+         platform_list: A list containing the platform(s) that our data has been
+             run on. (command line, firefox, chrome, etc)
+         variants: A list specifying whether we hold data about Frog
+             generated code, plain JS code, or a combination of both, or
+             Dart depending on the test.
+         values_list: A list containing the type of data we will be graphing
+             (benchmarks, percentage passing, etc).
+         test_runner: Reference to the parent test runner object that notifies a
+             test when to run.
+         tester: The visitor that actually performs the test running mechanics.
+         file_processor: The visitor that processes files in the format
+             appropriate for this test.
+         grapher: The visitor that generates graphs given our test result data.
+         extra_metrics: A list of any additional measurements we wish to keep
+             track of (such as the geometric mean of a set, the sum, etc).
+         build_targets: The targets necessary to build to run these tests
+             (default target is create_sdk)."""
     self.result_folder_name = result_folder_name
     # cur_time is used as a timestamp of when this performance test was run.
     self.cur_time = str(time.mktime(datetime.datetime.now().timetuple()))
-    # TODO(vsm): Factor out.
-    self.browser_color = {'chrome': 'green', 'ie': 'blue', 'ff': 'red',
-        'safari':'black'}
     self.values_list = values_list
     self.platform_list = platform_list
     self.revision_dict = dict()
     self.values_dict = dict()
-    self.color_index = 0
+    self.test_runner = test_runner
+    self.tester = tester
+    self.file_processor = file_processor
+    self.grapher = grapher
+    self.extra_metrics = extra_metrics
+    self.build_targets = build_targets
+    # Initialize our values store.
     for platform in platform_list:
       self.revision_dict[platform] = dict()
       self.values_dict[platform] = dict()
       for f in variants:
         self.revision_dict[platform][f] = dict()
         self.values_dict[platform][f] = dict()
         for val in values_list:
           self.revision_dict[platform][f][val] = []
           self.values_dict[platform][f][val] = []
-        self.revision_dict[platform][f][GEO_MEAN] = []
-        self.values_dict[platform][f][GEO_MEAN] = []
+        for extra_metric in extra_metrics:
+          self.revision_dict[platform][f][extra_metric] = []
+          self.values_dict[platform][f][extra_metric] = []
+  
+  def run(self, graph_only):
+    """Run the benchmarks/tests from the command line and plot the
+    results.
+    
+    Args:
+      graph_only: True if we should just graph the results instead of also
+          running tests."""
+    for visitor in [self.tester, self.file_processor, self.grapher]:
+      visitor.prepare()
+    
+    os.chdir(TestRunner.DART_INSTALL_LOCATION)
+    self.test_runner.ensure_output_directory(self.result_folder_name)
+    if not graph_only:
+      self.tester.run_tests()
 
-  def get_color(self):
-    color = COLORS[self.color_index]
-    self.color_index = (self.color_index + 1) % len(COLORS)
-    return color
+    os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
+
+    # TODO(efortuna): You will want to make this only use a subset of the files
+    # eventually.
+    files = os.listdir(self.result_folder_name)
+
+    for afile in files:
+      if not afile.startswith('.'):
+        self.file_processor.process_file(afile)
+
+    if 'plt' in globals():
+      # Only run Matplotlib if it is installed.
+      self.grapher.plot_results('%s.png' % self.result_folder_name)
+
+
+class Tester(object):
+  """The base level visitor class that runs tests. It contains convenience 
+  methods that many Tester objects use. Any class that would like to be a
+  TesterVisitor must implement the run_tests() method."""
+
+  def __init__(self, test):
+    self.test = test
+
+  def prepare(self):
+    """Perform any initial setup required before the test is run."""
+    pass
+
+  def add_svn_revision_to_trace(self, outfile):
+    """Add the svn version number to the provided tracefile."""
+    def search_for_revision(svn_info_command):
+      p = subprocess.Popen(svn_info_command, stdout = subprocess.PIPE,
+                           stderr = subprocess.STDOUT, shell =
+                           self.test.test_runner.has_shell)
+      output, _ = p.communicate()
+      for line in output.split('\n'):
+        if 'Revision' in line:
+          self.test.test_runner.run_cmd(['echo', line.strip()], outfile)
+          return True
+      return False
+
+    if not search_for_revision(['svn', 'info']):
+      if not search_for_revision(['git', 'svn', 'info']):
+        self.test.test_runner.run_cmd(['echo', 'Revision: unknown'], outfile)
+
+
+class Processor(object):
+  """The base level vistor class that processes tests. It contains convenience 
+  methods that many File Processor objects use. Any class that would like to be
+  a ProcessorVisitor must implement the process_file() method."""
+
+  def __init__(self, test):
+    self.test = test
+
+  def prepare(self):
+    """Perform any initial setup required before the test is run."""
+    pass
+
+  def calculate_geometric_mean(self, platform, variant, svn_revision):
+    """Calculate the aggregate geometric mean for JS and frog benchmark sets,
+    given two benchmark dictionaries."""
+    geo_mean = 0
+    for benchmark in self.test.values_list:
+      geo_mean += math.log(self.test.values_dict[platform][variant][benchmark][
+          len(self.test.values_dict[platform][variant][benchmark]) - 1])
+
+    self.test.values_dict[platform][variant]['Geo-Mean'] += \
+        [math.pow(math.e, geo_mean / len(self.test.values_list))]
+    self.test.revision_dict[platform][variant]['Geo-Mean'] += [svn_revision]
+
+
+class Grapher(object):
+  """The base level visitor class that generates graphs for data. It contains 
+  convenience methods that many Grapher objects use. Any class that would like
+  to be a GrapherVisitor must implement the plot_results() method."""
+  
+  graph_out_dir = 'graphs'
+  
+  def __init__(self, test):
+    self.color_index = 0
+    self.test = test
+  
+  def prepare(self):
+    """Perform any initial setup required before the test is run."""
+    if 'plt' in globals():
+      plt.cla() # cla = clear current axes
+    else:
+      print 'Unable to import Matplotlib and therefore unable to generate ' + \
+          'graphs. Please install it for this version of Python.'
+    self.test.test_runner.ensure_output_directory(Grapher.graph_out_dir)
 
   def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y,
-      legend_loc, filename, platform_list, variants, values_list,
-      should_clear_axes=True):
+                               legend_loc, filename, platform_list, variants, 
+                               values_list, should_clear_axes=True):
     """Sets style preferences for chart boilerplate that is consistent across
     all charts, and saves the chart as a png.
 
     Args:
       size_x: the size of the printed chart, in inches, in the horizontal
         direction
       size_y: the size of the printed chart, in inches in the vertical direction
       legend_loc: the location of the legend in on the chart. See suitable
         arguments for the loc argument in matplotlib
       filename: the filename that we want to save the resulting chart as
       platform_list: a list containing the platform(s) that our data has been
         run on. (command line, firefox, chrome, etc)
       values_list: a list containing the type of data we will be graphing
         (performance, percentage passing, etc)
       should_clear_axes: True if we want to create a fresh graph, instead of
         plotting additional lines on the current graph."""
     if should_clear_axes:
       plt.cla() # cla = clear current axes
     for platform in platform_list:
       for f in variants:
         for val in values_list:
-          plt.plot(self.revision_dict[platform][f][val],
-              self.values_dict[platform][f][val],
+          plt.plot(self.test.revision_dict[platform][f][val],
+              self.test.values_dict[platform][f][val],
               color=self.get_color(), label='%s-%s-%s' % (platform, f, val))
 
     plt.xlabel('Revision Number')
     plt.ylabel(y_axis_label)
     plt.title(chart_title)
     fontP = FontProperties()
     fontP.set_size('small')
     plt.legend(loc=legend_loc, prop = fontP)
 
     fig = plt.gcf()
     fig.set_size_inches(size_x, size_y)
-    fig.savefig(os.path.join(GRAPH_OUT_DIR, filename))
-
-  def add_svn_revision_to_trace(self, outfile):
-    """Add the svn version number to the provided tracefile."""
-    def search_for_revision(svn_info_command):
-      p = subprocess.Popen(svn_info_command, stdout = subprocess.PIPE,
-                           stderr = subprocess.STDOUT, shell = HAS_SHELL)
-      output, _ = p.communicate()
-      for line in output.split('\n'):
-        if 'Revision' in line:
-          run_cmd(['echo', line.strip()], outfile)
-          return True
-      return False
-
-    if not search_for_revision(['svn', 'info']):
-      if not search_for_revision(['git', 'svn', 'info']):
-        run_cmd(['echo', 'Revision: unknown'], outfile)
-
-  def calculate_geometric_mean(self, platform, variant, svn_revision):
-    """Calculate the aggregate geometric mean for JS and frog benchmark sets,
-    given two benchmark dictionaries."""
-    geo_mean = 0
-    for benchmark in self.values_list:
-      geo_mean += math.log(self.values_dict[platform][variant][benchmark][
-          len(self.values_dict[platform][variant][benchmark]) - 1])
-
-    self.values_dict[platform][variant][GEO_MEAN] += \
-        [math.pow(math.e, geo_mean / len(self.values_list))]
-    self.revision_dict[platform][variant][GEO_MEAN] += [svn_revision]
-
-  def run(self, graph_only):
-    """Run the benchmarks/tests from the command line and plot the
-    results."""
-    plt.cla() # cla = clear current axes
-    os.chdir(DART_INSTALL_LOCATION)
-    ensure_output_directory(self.result_folder_name)
-    ensure_output_directory(GRAPH_OUT_DIR)
-    if not graph_only:
-      self.run_tests()
-
-    os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
-
-    # TODO(efortuna): You will want to make this only use a subset of the files
-    # eventually.
-    files = os.listdir(self.result_folder_name)
-
-    for afile in files:
-      if not afile.startswith('.'):
-        self.process_file(afile)
-
-    if 'plt' in globals():
-      # Only run Matplotlib if it is installed.
-      self.plot_results('%s.png' % self.result_folder_name)
-    else:
-      print 'Unable to import Matplotlib and therefore unable to generate ' + \
-          'graphs. Please install it for this version of Python.'
-
-class PerformanceTest(TestRunner):
-  """Super class for all performance testing."""
+    fig.savefig(os.path.join(Grapher.graph_out_dir, filename))
+  
+  def get_color(self):
+    # Just a bunch of distinct colors for a potentially large number of values
+    # we wish to graph.

[vsm, 2012/04/09 20:19:58]

:-)

+    colors = [
+        '#3366CC', '#DC3912', '#FF9900', '#109618', '#990099', '#0099C6', 
+        '#DD4477', '#66AA00', '#B82E2E', '#316395', '#994499', '#22AA99',
+        '#AAAA11', '#6633CC', '#E67300', '#8B0707', '#651067', '#329262',
+        '#5574A6', '#3B3EAC', '#B77322', '#16D620', '#B91383', '#F4359E',
+        '#9C5935', '#A9C413', '#2A778D', '#668D1C', '#BEA413', '#0C5922',
+        '#743411', '#45AFE2', '#FF3300', '#FFCC00', '#14C21D', '#DF51FD',
+        '#15CBFF', '#FF97D2', '#97FB00', '#DB6651', '#518BC6', '#BD6CBD', 
+        '#35D7C2', '#E9E91F', '#9877DD', '#FF8F20', '#D20B0B', '#B61DBA', 
+        '#40BD7E', '#6AA7C4', '#6D70CD', '#DA9136', '#2DEA36', '#E81EA6',
+        '#F558AE', '#C07145', '#D7EE53', '#3EA7C6', '#97D129', '#E9CA1D', 
+        '#149638', '#C5571D']
+    color = colors[self.color_index]
+    self.color_index = (self.color_index + 1) % len(colors)
+    return color
+
+
+class RuntimePerformanceTest(Test):
+  """Super class for all runtime performance testing."""
+
   def __init__(self, result_folder_name, platform_list, platform_type,
-               versions, benchmarks):
-    super(PerformanceTest, self).__init__(result_folder_name,
-        platform_list, versions, benchmarks)
+               versions, benchmarks, test_runner, tester, file_processor,
+               build_targets=['create_sdk']):
+    """Args:
+        result_folder_name: The name of the folder where a tracefile of
+            performance results will be stored.
+        platform_list: A list containing the platform(s) that our data has been
+            run on. (command line, firefox, chrome, etc)
+        variants: A list specifying whether we hold data about Frog
+            generated code, plain JS code, or a combination of both, or
+            Dart depending on the test.
+        values_list: A list containing the type of data we will be graphing
+            (benchmarks, percentage passing, etc).
+        test_runner: Reference to the parent test runner object that notifies a
+            test when to run.
+        tester: The visitor that actually performs the test running mechanics.
+        file_processor: The visitor that processes files in the format
+            appropriate for this test.
+        grapher: The visitor that generates graphs given our test result data.
+        extra_metrics: A list of any additional measurements we wish to keep
+            track of (such as the geometric mean of a set, the sum, etc).
+        build_targets: The targets necessary to build to run these tests
+            (default target is create_sdk)."""
+    super(RuntimePerformanceTest, self).__init__(result_folder_name,
+          platform_list, versions, benchmarks, test_runner, tester,
+          file_processor, RuntimePerformanceTest.RuntimePerfGrapher(self),
+          build_targets=build_targets)
     self.platform_list = platform_list
     self.platform_type = platform_type
     self.versions = versions
-    self.benchmarks = benchmarks
-
-  def plot_all_perf(self, png_filename):
-    """Create a plot that shows the performance changes of individual benchmarks
-    run by JS and generated by frog, over svn history."""
-    for benchmark in self.benchmarks:
-      self.style_and_save_perf_plot(
-          'Performance of %s over time on the %s on %s' % (benchmark,
-          self.platform_type, utils.GuessOS()), 'Speed (bigger = better)', 16,
-          14, 'lower left', benchmark + png_filename, self.platform_list,
-          self.versions, [benchmark])
-
-  def plot_avg_perf(self, png_filename):
-    """Generate a plot that shows the performance changes of the geomentric mean
-    of JS and frog benchmark performance over svn history."""
-    (title, y_axis, size_x, size_y, loc, filename) = \
-        ('Geometric Mean of benchmark %s performance on %s ' %
-        (self.platform_type, utils.GuessOS()), 'Speed (bigger = better)', 16, 5,
-        'lower left', 'avg'+png_filename)
-    clear_axis = True
-    for platform in self.platform_list:
-      for version in self.versions:
-        self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc,
-                                      filename, [platform], [version],
-                                      [GEO_MEAN], clear_axis)
-        clear_axis = False
-
-  def plot_results(self, png_filename):
-    self.plot_all_perf(png_filename)
-    self.plot_avg_perf('2' + png_filename)
-
-
-class CommandLinePerformanceTest(PerformanceTest):
+    self.benchmarks = benchmarks 
+ 
+  class RuntimePerfGrapher(Grapher):
+    def plot_all_perf(self, png_filename):
+      """Create a plot that shows the performance changes of individual
+      benchmarks run by JS and generated by frog, over svn history."""
+      for benchmark in self.test.benchmarks:
+        self.style_and_save_perf_plot(
+            'Performance of %s over time on the %s on %s' % (benchmark,
+            self.test.platform_type, utils.GuessOS()), 
+            'Speed (bigger = better)', 16, 14, 'lower left',
+            benchmark + png_filename, self.test.platform_list,
+            self.test.versions, [benchmark])
+
+    def plot_avg_perf(self, png_filename):
+      """Generate a plot that shows the performance changes of the geomentric
+      mean of JS and frog benchmark performance over svn history."""
+      (title, y_axis, size_x, size_y, loc, filename) = \
+          ('Geometric Mean of benchmark %s performance on %s ' %
+          (self.test.platform_type, utils.GuessOS()), 'Speed (bigger = better)',
+          16, 5, 'lower left', 'avg'+png_filename)
+      clear_axis = True
+      for platform in self.test.platform_list:
+        for version in self.test.versions:
+          for metric in self.test.extra_metrics:
+            self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc,
+                                          filename, [platform], [version],
+                                          [metric], clear_axis)
+
+    def plot_results(self, png_filename):
+      self.plot_all_perf(png_filename)
+      self.plot_avg_perf('2' + png_filename)
+
+
+class BenchpressAndCoCommandLineTest(RuntimePerformanceTest):

[vsm, 2012/04/09 20:19:58]

"BenchpressAndCo" is a mouthful (fingerpressfull?).  Maybe "Core" or "Common" or
something like that?

[Emily Fortuna, 2012/04/09 21:15:03]

Done.

   """Run performance tests from the command line."""
 
-  def __init__(self):
-    super(CommandLinePerformanceTest, self).__init__(
-        CL_PERF, [COMMAND_LINE], 'command line',
-        JS_AND_FROG, get_standalone_benchmarks())
-
-  def process_file(self, afile):
-    """Pull all the relevant information out of a given tracefile.
-
-    Args:
-      afile: The filename string we will be processing."""
-    os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
-        'perf_testing'))
-    f = open(os.path.join(self.result_folder_name, afile))
-    tabulate_data = False
-    revision_num = 0
-    for line in f.readlines():
-      if 'Revision' in line:
-        revision_num = int(line.split()[1])
-      elif 'Benchmark' in line:
-        tabulate_data = True
-      elif tabulate_data:
-        tokens = line.split()
-        if len(tokens) < 4 or tokens[0] not in self.benchmarks:
-          #Done tabulating data.
-          break
-        js_value = float(tokens[1])
-        frog_value = float(tokens[3])
-        if js_value == 0 or frog_value == 0:
-          #Then there was an error when this performance test was run. Do not
-          #count it in our numbers.
-          return
-        benchmark = tokens[0]
-        self.revision_dict[COMMAND_LINE][JS][benchmark] += [revision_num]
-        self.values_dict[COMMAND_LINE][JS][benchmark] += [js_value]
-        self.revision_dict[COMMAND_LINE][FROG][benchmark] += [revision_num]
-        self.values_dict[COMMAND_LINE][FROG][benchmark] += [frog_value]
-    f.close()
-
-    self.calculate_geometric_mean(COMMAND_LINE, FROG, revision_num)
-    self.calculate_geometric_mean(COMMAND_LINE, JS, revision_num)
-
-  def run_tests(self):
-    """Run a performance test on our updated system."""
-    os.chdir('frog')
-    self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing',
-        self.result_folder_name, 'result' + self.cur_time)
-    run_cmd(['python', os.path.join('benchmarks', 'perf_tests.py')],
-        self.trace_file)
-    os.chdir('..')
-
-
-class BrowserStandalonePerformanceTest(PerformanceTest):
+  def __init__(self, test_runner):
+    """Args:
+      test_runner: Reference to the object that notfies this test when to
+          run."""
+    super(BenchpressAndCoCommandLineTest, self).__init__(
+          BenchpressAndCoCommandLineTest.name(), ['commandline'], 

[vsm, 2012/04/09 20:19:58]

I could be wrong, but I think you could write self.name(), etc., here.

[Emily Fortuna, 2012/04/09 21:15:03]

Done.

+          'command line', ['js', 'frog'], 
+          BenchpressAndCoCommandLineTest.get_standalone_benchmarks(), 
+          test_runner,
+          BenchpressAndCoCommandLineTest.BenchpressAndCoCommandLineTester(self),
+          BenchpressAndCoCommandLineTest
+              .BenchpressAndCoCommandLineFileProcessor(self),
+          build_targets=['create_sdk', 'dart2js'])
+
+  @staticmethod
+  def name():
+    return 'cl-perf'
+
+  @staticmethod
+  def get_standalone_benchmarks():
+    return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees',
+    'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute',
+    'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers',
+    'TreeSort']
+
+  class BenchpressAndCoCommandLineTester(Tester):
+    def run_tests(self):
+      """Run a performance test on our updated system."""
+      os.chdir('frog')
+      self.test.trace_file = os.path.join(
+          '..', 'tools', 'testing', 'perf_testing', 
+          self.test.result_folder_name, 'result' + self.test.cur_time)
+      self.test.test_runner.run_cmd(['python', os.path.join('benchmarks', 
+                                     'perf_tests.py')], self.test.trace_file)
+      os.chdir('..')
+
+  class BenchpressAndCoCommandLineFileProcessor(Processor):
+    def process_file(self, afile):
+      """Pull all the relevant information out of a given tracefile.
+
+      Args:
+        afile: The filename string we will be processing."""
+      os.chdir(os.path.join(TestRunner.DART_INSTALL_LOCATION, 'tools',
+          'testing', 'perf_testing'))
+      f = open(os.path.join(self.test.result_folder_name, afile))
+      tabulate_data = False
+      revision_num = 0
+      for line in f.readlines():
+        if 'Revision' in line:
+          revision_num = int(line.split()[1])
+        elif 'Benchmark' in line:
+          tabulate_data = True
+        elif tabulate_data:
+          tokens = line.split()
+          if len(tokens) < 4 or tokens[0] not in self.test.benchmarks:
+            #Done tabulating data.
+            break
+          js_value = float(tokens[1])
+          frog_value = float(tokens[3])
+          if js_value == 0 or frog_value == 0:
+            #Then there was an error when this performance test was run. Do not
+            #count it in our numbers.
+            return
+          benchmark = tokens[0]
+          self.test.revision_dict['commandline']['js'][benchmark] += \
+              [revision_num]
+          self.test.values_dict['commandline']['js'][benchmark] += [js_value]
+          self.test.revision_dict['commandline']['frog'][benchmark] += \
+              [revision_num]
+          self.test.values_dict['commandline']['frog'][benchmark] += \
+              [frog_value]
+      f.close()
+
+      self.calculate_geometric_mean('commandline', 'frog', revision_num)
+      self.calculate_geometric_mean('commandline', 'js', revision_num)
+
+
+class BrowserTester(Tester):
+  # TODO(vsm): Add Dartium.
+  @staticmethod
+  def get_browsers():
+    browsers = ['ff', 'chrome']
+    if platform.system() == 'Darwin':
+      browsers += ['safari']
+    if platform.system() == 'Windows':
+      browsers += ['ie']
+    return browsers
+
+
+class BenchpressAndCoBrowserTest(RuntimePerformanceTest):
   """Runs standalone performance tests, in the browser."""
 
-  def __init__(self):
-    super(BrowserStandalonePerformanceTest, self).__init__(
-        BROWSER_PERF, get_browsers(), 'browser',
-        JS_AND_FROG, get_standalone_benchmarks())
-
-  def run_tests(self):
-    """Run a performance test in the browser."""
-
-    os.chdir('frog')
-    run_cmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')])
-    os.chdir('..')
-
-    for browser in get_browsers():
-      for version in self.versions:
-        self.trace_file = os.path.join('tools', 'testing', 'perf_testing',
-            self.result_folder_name,
-            'perf-%s-%s-%s' % (self.cur_time, browser, version))
-        self.add_svn_revision_to_trace(self.trace_file)
-        file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks',
-            'benchmark_page_%s.html' % version)
-        run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'),
-            '--out', file_path, '--browser', browser,
-            '--timeout', '600', '--mode', 'perf'], self.trace_file, append=True)
-
-  def process_file(self, afile):
-    """Comb through the html to find the performance results."""
-    os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
-        'perf_testing'))
-    parts = afile.split('-')
-    browser = parts[2]
-    version = parts[3]
-    f = open(os.path.join(self.result_folder_name, afile))
-    lines = f.readlines()
-    line = ''
-    i = 0
-    revision_num = 0
-    while '<div id="results">' not in line and i < len(lines):
-      if 'Revision' in line:
-        revision_num = int(line.split()[1].strip('"'))
+  def __init__(self, test_runner):
+    """Args:
+      test_runner: Reference to the object that notifies us when to run."""
+    super(BenchpressAndCoBrowserTest, self).__init__(
+        BenchpressAndCoBrowserTest.name(), BrowserTester.get_browsers(),
+        'browser', ['js', 'frog'],
+        BenchpressAndCoBrowserTest.get_standalone_benchmarks(), test_runner, 
+        BenchpressAndCoBrowserTest.BenchpressAndCoBrowserTester(self),
+        BenchpressAndCoBrowserTest.BenchpressAndCoBrowserFileProcessor(self))
+  
+  @staticmethod
+  def name():
+    return 'browser-perf'
+
+  @staticmethod
+  def get_standalone_benchmarks():
+    return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees',
+    'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute',
+    'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers',
+    'TreeSort']
+
+  class BenchpressAndCoBrowserTester(BrowserTester):
+    def run_tests(self):
+      """Run a performance test in the browser."""
+      os.chdir('frog')
+      self.test.test_runner.run_cmd(['python', os.path.join('benchmarks', 
+                                     'make_web_benchmarks.py')])
+      os.chdir('..')
+
+      for browser in BrowserTester.get_browsers():
+        for version in self.test.versions:
+          self.test.trace_file = os.path.join(
+              'tools', 'testing', 'perf_testing', self.test.result_folder_name,
+              'perf-%s-%s-%s' % (self.test.cur_time, browser, version))
+          self.add_svn_revision_to_trace(self.test.trace_file)
+          file_path = os.path.join(
+              os.getcwd(), 'internal', 'browserBenchmarks',
+              'benchmark_page_%s.html' % version)
+          self.test.test_runner.run_cmd(
+              ['python', os.path.join('tools', 'testing', 'run_selenium.py'),
+              '--out', file_path, '--browser', browser,
+              '--timeout', '600', '--mode', 'perf'], self.test.trace_file, 
+              append=True)
+
+  class BenchpressAndCoBrowserFileProcessor(Processor):
+    def process_file(self, afile):
+      """Comb through the html to find the performance results."""
+      os.chdir(os.path.join(TestRunner.DART_INSTALL_LOCATION, 'tools',
+                            'testing', 'perf_testing'))
+      parts = afile.split('-')
+      browser = parts[2]
+      version = parts[3]
+      f = open(os.path.join(self.test.result_folder_name, afile))
+      lines = f.readlines()
+      line = ''
+      i = 0
+      revision_num = 0
+      while '<div id="results">' not in line and i < len(lines):
+        if 'Revision' in line:
+          revision_num = int(line.split()[1].strip('"'))
+        line = lines[i]
+        i += 1
+
+      if i >= len(lines) or revision_num == 0:
+        # Then this run did not complete. Ignore this tracefile.
+        return
+
       line = lines[i]
       i += 1
-
-    if i >= len(lines) or revision_num == 0:
-      # Then this run did not complete. Ignore this tracefile.
-      return
-
-    line = lines[i]
-    i += 1
-    results = []
-    if line.find('<br>') > -1:
-      results = line.split('<br>')
-    else:
-      results = line.split('<br />')
-    for result in results:
-      name_and_score = result.split(':')
-      if len(name_and_score) < 2:
-        break
-      name = name_and_score[0].strip()
-      score = name_and_score[1].strip()
-      if version == JS or version == 'v8':
-        version = JS
-        bench_dict = self.values_dict[browser][JS]
+      results = []
+      if line.find('<br>') > -1:
+        results = line.split('<br>')
       else:
-        bench_dict = self.values_dict[browser][FROG]
-      bench_dict[name] += [float(score)]
-      self.revision_dict[browser][version][name] += [revision_num]
-
-    f.close()
-    self.calculate_geometric_mean(browser, version, revision_num)
-
-
-# TODO(vsm): This should not be hardcoded here if possible.
-DROMAEO_BENCHMARKS = {
-    'attr': ('attributes', [
-        'getAttribute',
-        'element.property',
-        'setAttribute',
-        'element.property = value']),
-    'modify': ('modify', [
-        'createElement',
-        'createTextNode',
-        'innerHTML',
-        'cloneNode',
-        'appendChild',
-        'insertBefore']),
-    'query': ('query', [
-        'getElementById',
-        'getElementById (not in document)',
-        'getElementsByTagName(div)',
-        'getElementsByTagName(p)',
-        'getElementsByTagName(a)',
-        'getElementsByTagName(*)',
-        'getElementsByTagName (not in document)',
-        'getElementsByName',
-        'getElementsByName (not in document)']),
-    'traverse': ('traverse', [
-        'firstChild',
-        'lastChild',
-        'nextSibling',
-        'previousSibling',
-        'childNodes'])
-}
-
-# Use legal appengine filenames for benchmark names.
-def legalize_filename(str):
-  remap = {
-      ' ': '_',
-      '(': '_',
-      ')': '_',
-      '*': 'ALL',
-      '=': 'ASSIGN',
-      }
-  for (old, new) in remap.iteritems():
-    str = str.replace(old, new)
-  return str
-
-# TODO(vsm): This is a hack to skip breaking tests.  Triage this
-# failure properly.  The modify suite fails on 32-bit chrome on
-# the mac.
-def get_valid_dromaeo_tags():
-  tags = [tag for (tag, _) in DROMAEO_BENCHMARKS.values()]
-  if platform.system() == 'Darwin':
-    tags.remove('modify')
-  return tags
-
-def get_dromaeo_benchmarks():
-  valid = get_valid_dromaeo_tags()
-  benchmarks = reduce(lambda l1,l2: l1+l2,
-                      [tests for (tag, tests) in
-                       DROMAEO_BENCHMARKS.values() if tag in valid])
-  return map(legalize_filename, benchmarks)
-
-def get_dromaeo_versions():
-  return ['js', 'frog_dom', 'frog_html']
-
-def get_dromaeo_url_query(version):
-  version = version.replace('_','&')
-  tags = get_valid_dromaeo_tags()
-  return '|'.join([ '%s&%s' % (version, tag) for tag in tags])
-
-class DromaeoTest(PerformanceTest):
+        results = line.split('<br />')
+      for result in results:
+        name_and_score = result.split(':')
+        if len(name_and_score) < 2:
+          break
+        name = name_and_score[0].strip()
+        score = name_and_score[1].strip()
+        if version == 'js' or version == 'v8':
+          version = 'js'
+          bench_dict = self.test.values_dict[browser]['js']
+        else:
+          bench_dict = self.test.values_dict[browser]['frog']
+        bench_dict[name] += [float(score)]
+        self.test.revision_dict[browser][version][name] += [revision_num]
+
+      f.close()
+      self.calculate_geometric_mean(browser, version, revision_num)
+
+class DromaeoTester(Tester):
+  DROMAEO_BENCHMARKS = {
+      'attr': ('attributes', [
+          'getAttribute',
+          'element.property',
+          'setAttribute',
+          'element.property = value']),
+      'modify': ('modify', [
+          'createElement',
+          'createTextNode',
+          'innerHTML',
+          'cloneNode',
+          'appendChild',
+          'insertBefore']),
+      'query': ('query', [
+          'getElementById',
+          'getElementById (not in document)',
+          'getElementsByTagName(div)',
+          'getElementsByTagName(p)',
+          'getElementsByTagName(a)',
+          'getElementsByTagName(*)',
+          'getElementsByTagName (not in document)',
+          'getElementsByName',
+          'getElementsByName (not in document)']),
+      'traverse': ('traverse', [
+          'firstChild',
+          'lastChild',
+          'nextSibling',
+          'previousSibling',
+          'childNodes'])
+  }
+
+  # Use legal appengine filenames for benchmark names.
+  @staticmethod
+  def legalize_filename(str):
+    remap = {
+        ' ': '_',
+        '(': '_',
+        ')': '_',
+        '*': 'ALL',
+        '=': 'ASSIGN',
+        }
+    for (old, new) in remap.iteritems():
+      str = str.replace(old, new)
+    return str
+
+  # TODO(vsm): This is a hack to skip breaking tests.  Triage this
+  # failure properly.  The modify suite fails on 32-bit chrome on
+  # the mac.
+  @staticmethod
+  def get_valid_dromaeo_tags():
+    tags = [tag for (tag, _) in DromaeoTester.DROMAEO_BENCHMARKS.values()]
+    if platform.system() == 'Darwin':
+      tags.remove('modify')
+    return tags
+
+  @staticmethod
+  def get_dromaeo_benchmarks():
+    valid = DromaeoTester.get_valid_dromaeo_tags()
+    benchmarks = reduce(lambda l1,l2: l1+l2,
+                        [tests for (tag, tests) in
+                         DromaeoTester.DROMAEO_BENCHMARKS.values() 
+                         if tag in valid])
+    return map(DromaeoTester.legalize_filename, benchmarks)
+
+  @staticmethod
+  def get_dromaeo_versions():
+    return ['js', 'frog_dom', 'frog_html']
+
+
+class DromaeoTest(RuntimePerformanceTest):
   """Runs Dromaeo tests, in the browser."""
-  def __init__(self):
+  def __init__(self, test_runner):
     super(DromaeoTest, self).__init__(
-        DROMAEO, get_browsers(), 'browser',
-        get_dromaeo_versions(), get_dromaeo_benchmarks())
-
-  def run_tests(self):
-    """Run dromaeo in the browser."""
-
-    # Build tests.
-    dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
-    current_path = os.getcwd()
-    os.chdir(dromaeo_path)
-    run_cmd(['python', 'generate_frog_tests.py'])
-    os.chdir(current_path)
-
-    versions = get_dromaeo_versions()
-
-    for browser in get_browsers():
-      for version_name in versions:
-        version = get_dromaeo_url_query(version_name)
-        self.trace_file = os.path.join('tools', 'testing', 'perf_testing',
-            self.result_folder_name,
-            'dromaeo-%s-%s-%s' % (self.cur_time, browser, version_name))
-        self.add_svn_revision_to_trace(self.trace_file)
-        file_path = os.path.join(os.getcwd(), dromaeo_path,
-            'index-js.html?%s' % version)
-        run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'),
-                 '--out', file_path, '--browser', browser,
-                 '--timeout', '200', '--mode', 'dromaeo'], self.trace_file,
-                append=True)
-
-  def process_file(self, afile):
-    """Comb through the html to find the performance results."""
-    parts = afile.split('-')
-    browser = parts[2]
-    version = parts[3]
-
-    bench_dict = self.values_dict[browser][version]
-
-    f = open(os.path.join(self.result_folder_name, afile))
-    lines = f.readlines()
-    i = 0
-    revision_num = 0
-    revision_pattern = r'Revision: (\d+)'
-    suite_pattern = r'<div class="result-item done">(.+?)</ol></div>'
-    result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)'
-
-    for line in lines:
-      rev = re.match(revision_pattern, line.strip())
-      if rev:
-        revision_num = int(rev.group(1))
-        continue
-
-      suite_results = re.findall(suite_pattern, line)
-      if suite_results:
-        for suite_result in suite_results:
-          results = re.findall(r'<li>(.*?)</li>', suite_result)
-          if results:
-            for result in results:
-              r = re.match(result_pattern, result)
-              name = legalize_filename(r.group(1).strip(':'))
-              score = float(r.group(2))
-              bench_dict[name] += [float(score)]
-              self.revision_dict[browser][version][name] += [revision_num]
-
-    f.close()
-    self.calculate_geometric_mean(browser, version, revision_num)
-
-
-class DromaeoSizeTest(TestRunner):
+        DromaeoTest.name(), BrowserTester.get_browsers(), 'browser',
+        DromaeoTester.get_dromaeo_versions(), 
+        DromaeoTester.get_dromaeo_benchmarks(), test_runner,
+        DromaeoTest.DromaeoPerfTester(self),
+        DromaeoTest.DromaeoFileProcessor(self))
+
+  @staticmethod
+  def name():
+    return 'dromaeo'
+
+  class DromaeoPerfTester(DromaeoTester):
+    def run_tests(self):
+      """Run dromaeo in the browser."""
+
+      # Build tests.
+      dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
+      current_path = os.getcwd()
+      os.chdir(dromaeo_path)
+      self.test.test_runner.run_cmd(['python', 'generate_frog_tests.py'])
+      os.chdir(current_path)
+
+      versions = DromaeoTester.get_dromaeo_versions()
+
+      for browser in BrowserTester.get_browsers():
+        for version_name in versions:
+          version = DromaeoTest.DromaeoPerfTester.get_dromaeo_url_query(
+              version_name)
+          self.test.trace_file = os.path.join(
+              'tools', 'testing', 'perf_testing', self.test.result_folder_name,
+              'dromaeo-%s-%s-%s' % (self.test.cur_time, browser, version_name))
+          self.add_svn_revision_to_trace(self.test.trace_file)
+          file_path = os.path.join(os.getcwd(), dromaeo_path,
+              'index-js.html?%s' % version)
+          self.test.test_runner.run_cmd(
+              ['python', os.path.join('tools', 'testing', 'run_selenium.py'),
+               '--out', file_path, '--browser', browser,
+               '--timeout', '200', '--mode', 'dromaeo'], self.test.trace_file,
+               append=True)
+
+    @staticmethod
+    def get_dromaeo_url_query(version):
+      version = version.replace('_','&')
+      tags = DromaeoTester.get_valid_dromaeo_tags()
+      return '|'.join([ '%s&%s' % (version, tag) for tag in tags])
+
+
+  class DromaeoFileProcessor(Processor):
+    def process_file(self, afile):
+      """Comb through the html to find the performance results."""
+      parts = afile.split('-')
+      browser = parts[2]
+      version = parts[3]
+
+      bench_dict = self.test.values_dict[browser][version]
+
+      f = open(os.path.join(self.test.result_folder_name, afile))
+      lines = f.readlines()
+      i = 0
+      revision_num = 0
+      revision_pattern = r'Revision: (\d+)'
+      suite_pattern = r'<div class="result-item done">(.+?)</ol></div>'
+      result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)'
+
+      for line in lines:
+        rev = re.match(revision_pattern, line.strip())
+        if rev:
+          revision_num = int(rev.group(1))
+          continue
+
+        suite_results = re.findall(suite_pattern, line)
+        if suite_results:
+          for suite_result in suite_results:
+            results = re.findall(r'<li>(.*?)</li>', suite_result)
+            if results:
+              for result in results:
+                r = re.match(result_pattern, result)
+                name = DromaeoTester.legalize_filename(
+                    r.group(1).strip(':'))
+                score = float(r.group(2))
+                bench_dict[name] += [float(score)]
+                self.test.revision_dict[browser][version][name] += \
+                    [revision_num]
+
+      f.close()
+      self.calculate_geometric_mean(browser, version, revision_num)
+
+
+class DromaeoSizeTest(Test):
   """Run tests to determine the compiled file output size of Dromaeo."""
-  def __init__(self):
+  def __init__(self, test_runner):
     super(DromaeoSizeTest, self).__init__(
-        DROMAEO_SIZE,
+        DromaeoSizeTest.name(),
         ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
-        DROMAEO_BENCHMARKS.keys())
-
-  def run_tests(self):
-    # Build tests.
-    dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
-    current_path = os.getcwd()
-    os.chdir(dromaeo_path)
-    run_cmd(['python', os.path.join('generate_frog_tests.py')])
-    os.chdir(current_path)
-
-    self.trace_file = os.path.join('tools', 'testing', 'perf_testing',
-        self.result_folder_name, self.result_folder_name + self.cur_time)
-    self.add_svn_revision_to_trace(self.trace_file)
-
-    variants = [
-        ('frog_dom', ''),
-        ('frog_html', '-html'),
-        ('frog_htmlidiomatic', '-htmlidiomatic')]
-
-    test_path = os.path.join(dromaeo_path, 'tests')
-    frog_path = os.path.join(test_path, 'frog')
-    total_size = {}
-    for (variant, _) in variants:
-      total_size[variant] = 0
-    total_dart_size = 0
-    for suite in DROMAEO_BENCHMARKS.keys():
-      dart_size = 0
+        DromaeoTester.DROMAEO_BENCHMARKS.keys(), test_runner, 
+        DromaeoSizeTest.DromaeoSizeTester(self),
+        DromaeoSizeTest.DromaeoSizeProcessor(self),
+        DromaeoSizeTest.DromaeoSizeGrapher(self), extra_metrics=['sum'])
+  
+  @staticmethod
+  def name():
+    return 'dromaeo-size'
+
+
+  class DromaeoSizeTester(DromaeoTester):
+    def run_tests(self):
+      # Build tests.
+      dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
+      current_path = os.getcwd()
+      os.chdir(dromaeo_path)
+      self.test.test_runner.run_cmd(
+          ['python', os.path.join('generate_frog_tests.py')])
+      os.chdir(current_path)
+
+      self.test.trace_file = os.path.join(
+          'tools', 'testing', 'perf_testing', self.test.result_folder_name,
+          self.test.result_folder_name + self.test.cur_time)
+      self.add_svn_revision_to_trace(self.test.trace_file)
+
+      variants = [
+          ('frog_dom', ''),
+          ('frog_html', '-html'),
+          ('frog_htmlidiomatic', '-htmlidiomatic')]
+
+      test_path = os.path.join(dromaeo_path, 'tests')
+      frog_path = os.path.join(test_path, 'frog')
+      total_size = {}
+      for (variant, _) in variants:
+        total_size[variant] = 0
+      total_dart_size = 0
+      for suite in DromaeoTester.DROMAEO_BENCHMARKS.keys():
+        dart_size = 0
+        try:
+          dart_size = os.path.getsize(os.path.join(test_path,
+                                                   'dom-%s.dart' % suite))
+        except OSError:
+          pass #If compilation failed, continue on running other tests.
+
+        total_dart_size += dart_size
+        self.test.test_runner.run_cmd(
+            ['echo', 'Size (dart, %s): %s' % (suite, str(dart_size))],
+            self.test.trace_file, append=True)
+
+        for (variant, suffix) in variants:
+          name = 'dom-%s%s.dart.js' % (suite, suffix)
+          js_size = 0
+          try:
+            # TODO(vsm): Strip comments at least.  Consider compression.
+            js_size = os.path.getsize(os.path.join(frog_path, name))
+          except OSError:
+            pass #If compilation failed, continue on running other tests.
+
+          total_size[variant] += js_size
+          self.test.test_runner.run_cmd(
+              ['echo', 'Size (%s, %s): %s' % (variant, suite, str(js_size))],
+              self.test.trace_file, append=True)
+
+      self.test.test_runner.run_cmd(
+          ['echo', 'Size (dart, %s): %s' % (total_dart_size, 
+                                            self.test.extra_metrics[0])],
+          self.test.trace_file, append=True)
+      for (variant, _) in variants:
+        self.test.test_runner.run_cmd(
+            ['echo', 'Size (%s, %s): %s' % (variant, self.test.extra_metrics[0],
+                                            total_size[variant])],
+            self.test.trace_file, append=True)
+
+  class DromaeoSizeProcessor(Processor):
+    def process_file(self, afile):
+      """Pull all the relevant information out of a given tracefile.
+
+      Args:
+        afile: is the filename string we will be processing."""
+      os.chdir(os.path.join(TestRunner.DART_INSTALL_LOCATION, 'tools',
+          'testing', 'perf_testing'))
+      f = open(os.path.join(self.test.result_folder_name, afile))
+      tabulate_data = False
+      revision_num = 0
+      revision_pattern = r'Revision: (\d+)'
+      result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)'
+
+      for line in f.readlines():
+        rev = re.match(revision_pattern, line.strip())
+        if rev:
+          revision_num = int(rev.group(1))
+          continue
+
+        result = re.match(result_pattern, line.strip())
+        if result:
+          variant = result.group(1)
+          metric = result.group(2)
+          num = result.group(3)
+          if num.find('.') == -1:
+            num = int(num)
+          else:
+            num = float(num)
+          self.test.values_dict['browser'][variant][metric] += [num]
+          self.test.revision_dict['browser'][variant][metric] += [revision_num]
+
+      f.close()
+  class DromaeoSizeGrapher(Grapher):
+    def plot_results(self, png_filename):
+      self.style_and_save_perf_plot(
+          'Compiled Dromaeo Sizes',
+          'Size (in bytes)', 10, 10, 'lower left', png_filename,
+          ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
+          DromaeoTester.DROMAEO_BENCHMARKS.keys())
+
+      self.style_and_save_perf_plot(
+          'Compiled Dromaeo Sizes',
+          'Size (in bytes)', 10, 10, 'lower left', '2' + png_filename,
+          ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
+          [self.test.extra_metrics[0]])
+  
+
+class CompileTimeAndSizeTest(Test):
+  """Run tests to determine how long minfrog takes to compile, and the compiled
+  file output size of some benchmarking files."""
+  def __init__(self, test_runner):
+    """Reference to the test_runner object that notifies us when to begin
+    testing."""
+    super(CompileTimeAndSizeTest, self).__init__(
+        CompileTimeAndSizeTest.name(), ['commandline'], ['frog'], 
+        ['Compiling on Dart VM', 'Bootstrapping', 'minfrog', 'swarm', 'total'],
+        test_runner, CompileTimeAndSizeTest.CompileTester(self),
+        CompileTimeAndSizeTest.CompileProcessor(self),
+        CompileTimeAndSizeTest.CompileGrapher(self))
+    self.dart_compiler = os.path.join(
+        TestRunner.DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(),
+        'release', 'ia32'), 'dart-sdk', 'bin', 'frogc')
+    _suffix = ''
+    if platform.system() == 'Windows':
+      _suffix = '.exe'
+    self.dart_vm = os.path.join(
+        TestRunner.DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), 
+        'release', 'ia32'), 'dart-sdk', 'bin','dart' + _suffix)
+    self.failure_threshold = {
+        'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, 'minfrog' : 100, 
+        'swarm' : 100, 'total' : 100}
+
+  @staticmethod
+  def name():
+    return 'time-size'
+
+  class CompileTester(Tester):
+    def run_tests(self):
+      os.chdir('frog')
+      self.test.trace_file = os.path.join(
+          '..', 'tools', 'testing', 'perf_testing', 
+          self.test.result_folder_name,
+          self.test.result_folder_name + self.test.cur_time)
+
+      self.add_svn_revision_to_trace(self.test.trace_file)
+
+      elapsed = self.test.test_runner.time_cmd(
+          [self.test.dart_vm, os.path.join('.', 'minfrogc.dart'),
+          '--out=minfrog', 'minfrog.dart'])
+      self.test.test_runner.run_cmd(
+          ['echo', '%f Compiling on Dart VM in production mode in seconds'
+          % elapsed], self.test.trace_file, append=True)
+      elapsed = self.test.test_runner.time_cmd(
+          [os.path.join('.', 'minfrog'), '--out=minfrog', 'minfrog.dart', 
+          os.path.join('tests', 'hello.dart')])
+      if elapsed < self.test.failure_threshold['Bootstrapping']:
+        #minfrog didn't compile correctly. Stop testing now, because subsequent
+        #numbers will be meaningless.
+        return
+      size = os.path.getsize('minfrog')
+      self.test.test_runner.run_cmd(
+          ['echo', '%f Bootstrapping time in seconds in production mode' %
+          elapsed], self.test.trace_file, append=True)
+      self.test.test_runner.run_cmd(
+          ['echo', '%d Generated checked minfrog size' % size],
+          self.test.trace_file, append=True)
+
+      self.test.test_runner.run_cmd(
+          [self.test.dart_compiler, '--out=swarm-result',
+          os.path.join('..', 'samples', 'swarm',
+          'swarm.dart')])
+
+      swarm_size = 0
       try:
-        dart_size = os.path.getsize(os.path.join(test_path,
-                                                 'dom-%s.dart' % suite))
+        swarm_size = os.path.getsize('swarm-result')
       except OSError:
         pass #If compilation failed, continue on running other tests.
 
-      total_dart_size += dart_size
-      run_cmd(['echo', 'Size (dart, %s): %s' % (suite, str(dart_size))],
-              self.trace_file, append=True)
-
-      for (variant, suffix) in variants:
-        name = 'dom-%s%s.dart.js' % (suite, suffix)
-        js_size = 0
-        try:
-          # TODO(vsm): Strip comments at least.  Consider compression.
-          js_size = os.path.getsize(os.path.join(frog_path, name))
-        except OSError:
-          pass #If compilation failed, continue on running other tests.
-
-        total_size[variant] += js_size
-        run_cmd(['echo', 'Size (%s, %s): %s' % (variant, suite,
-                                                str(js_size))],
-                self.trace_file, append=True)
-
-    # TODO(vsm): Change GEO_MEAN to sum.  The base class assumes
-    # GEO_MEAN right now.
-    run_cmd(['echo', 'Size (dart, %s): %s' % (total_dart_size, GEO_MEAN)],
-            self.trace_file, append=True)
-    for (variant, _) in variants:
-      run_cmd(['echo', 'Size (%s, %s): %s' % (variant, GEO_MEAN,
-                                              total_size[variant])],
-              self.trace_file, append=True)
-
-
-  def process_file(self, afile):
-    """Pull all the relevant information out of a given tracefile.
-
-    Args:
-      afile: is the filename string we will be processing."""
-    os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
-        'perf_testing'))
-    f = open(os.path.join(self.result_folder_name, afile))
-    tabulate_data = False
-    revision_num = 0
-    revision_pattern = r'Revision: (\d+)'
-    result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)'
-
-    for line in f.readlines():
-      rev = re.match(revision_pattern, line.strip())
-      if rev:
-        revision_num = int(rev.group(1))
-        continue
-
-      result = re.match(result_pattern, line.strip())
-      if result:
-        variant = result.group(1)
-        metric = result.group(2)
-        num = result.group(3)
-        if num.find('.') == -1:
-          num = int(num)
+      self.test.test_runner.run_cmd(
+          [self.test.dart_compiler, '--out=total-result',
+          os.path.join('..', 'samples', 'total',
+          'client', 'Total.dart')])
+      total_size = 0
+      try:
+        total_size = os.path.getsize('total-result')
+      except OSError:
+        pass #If compilation failed, continue on running other tests.
+
+      self.test.test_runner.run_cmd(
+          ['echo', '%d Generated checked swarm size' % swarm_size],
+          self.test.trace_file, append=True)
+
+      self.test.test_runner.run_cmd(
+          ['echo', '%d Generated checked total size' % total_size],
+          self.test.trace_file, append=True)
+    
+      #Revert our newly built minfrog to prevent conflicts when we update
+      self.test.test_runner.run_cmd(
+          ['svn', 'revert',  os.path.join(os.getcwd(), 'frog', 'minfrog')])
+    
+      os.chdir('..')
+
+  class CompileProcessor(Processor):
+    def process_file(self, afile):
+      """Pull all the relevant information out of a given tracefile.
+
+      Args:
+        afile: is the filename string we will be processing."""
+      os.chdir(os.path.join(TestRunner.DART_INSTALL_LOCATION, 'tools',
+          'testing', 'perf_testing'))
+      f = open(os.path.join(self.test.result_folder_name, afile))
+      tabulate_data = False
+      revision_num = 0
+      for line in f.readlines():
+        tokens = line.split()
+        if 'Revision' in line:
+          revision_num = int(line.split()[1])
         else:
-          num = float(num)
-        self.values_dict['browser'][variant][metric] += [num]
-        self.revision_dict['browser'][variant][metric] += [revision_num]
-
-    f.close()
-
-  def plot_results(self, png_filename):
-    self.style_and_save_perf_plot(
-        'Compiled Dromaeo Sizes',
-        'Size (in bytes)', 10, 10, 'lower left', png_filename,
-        ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
-        DROMAEO_BENCHMARKS.keys())
-
-    self.style_and_save_perf_plot(
-        'Compiled Dromaeo Sizes',
-        'Size (in bytes)', 10, 10, 'lower left', '2' + png_filename,
-        ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
-        [GEO_MEAN])
-
-
-
-class CompileTimeAndSizeTest(TestRunner):
-  """Run tests to determine how long minfrog takes to compile, and the compiled
-  file output size of some benchmarking files."""
-  def __init__(self):
-    super(CompileTimeAndSizeTest, self).__init__(TIME_SIZE,
-        [COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping',
-        'minfrog', 'swarm', 'total'])
-    self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5,
-        'minfrog' : 100, 'swarm' : 100, 'total' : 100}
-
-  def run_tests(self):
-    os.chdir('frog')
-    self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing',
-        self.result_folder_name, self.result_folder_name + self.cur_time)
-
-    self.add_svn_revision_to_trace(self.trace_file)
-
-    elapsed = time_cmd([DART_VM, os.path.join('.', 'minfrogc.dart'),
-        '--out=minfrog', 'minfrog.dart'])
-    run_cmd(['echo', '%f Compiling on Dart VM in production mode in seconds'
-        % elapsed], self.trace_file, append=True)
-    elapsed = time_cmd([os.path.join('.', 'minfrog'), '--out=minfrog',
-        'minfrog.dart', os.path.join('tests', 'hello.dart')])
-    if elapsed < self.failure_threshold['Bootstrapping']:
-      #minfrog didn't compile correctly. Stop testing now, because subsequent
-      #numbers will be meaningless.
-      return
-    size = os.path.getsize('minfrog')
-    run_cmd(['echo', '%f Bootstrapping time in seconds in production mode' %
-        elapsed], self.trace_file, append=True)
-    run_cmd(['echo', '%d Generated checked minfrog size' % size],
-        self.trace_file, append=True)
-
-    run_cmd([DART_COMPILER, '--out=swarm-result',
-        os.path.join('..', 'samples', 'swarm',
-        'swarm.dart')])
-    swarm_size = 0
-    try:
-      swarm_size = os.path.getsize('swarm-result')
-    except OSError:
-      pass #If compilation failed, continue on running other tests.
-
-    run_cmd([DART_COMPILER, '--out=total-result',
-        os.path.join('..', 'samples', 'total',
-        'client', 'Total.dart')])
-    total_size = 0
-    try:
-      total_size = os.path.getsize('total-result')
-    except OSError:
-      pass #If compilation failed, continue on running other tests.
-
-    run_cmd(['echo', '%d Generated checked swarm size' % swarm_size],
-        self.trace_file, append=True)
-
-    run_cmd(['echo', '%d Generated checked total size' % total_size],
-        self.trace_file, append=True)
-    os.chdir('..')
-
-  def process_file(self, afile):
-    """Pull all the relevant information out of a given tracefile.
-
-    Args:
-      afile: is the filename string we will be processing."""
-    os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
-        'perf_testing'))
-    f = open(os.path.join(self.result_folder_name, afile))
-    tabulate_data = False
-    revision_num = 0
-    for line in f.readlines():
-      tokens = line.split()
-      if 'Revision' in line:
-        revision_num = int(line.split()[1])
-      else:
-        for metric in self.values_list:
-          if metric in line:
-            num = tokens[0]
-            if num.find('.') == -1:
-              num = int(num)
-            else:
-              num = float(num)
-            self.values_dict[COMMAND_LINE][FROG][metric] += [num]
-            self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
-
-    if revision_num != 0:
-      for metric in self.values_list:
-        self.revision_dict[COMMAND_LINE][FROG][metric].pop()
-        self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
-        # Fill in 0 if compilation failed.
-        if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \
-            self.failure_threshold[metric]:
-          self.values_dict[COMMAND_LINE][FROG][metric] += [0]
-          self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
-
-    f.close()
-
-  def plot_results(self, png_filename):
-    self.style_and_save_perf_plot('Compiled minfrog Sizes',
-        'Size (in bytes)', 10, 10, 'lower left', png_filename, [COMMAND_LINE],
-        [FROG], ['swarm', 'total', 'minfrog'])
-
-    self.style_and_save_perf_plot('Time to compile and bootstrap',
-        'Seconds', 10, 10, 'lower left', '2' + png_filename, [COMMAND_LINE],
-        [FROG], ['Bootstrapping', 'Compiling on Dart VM'])
-
-# TODO(vsm): Make these names consistent with BROWSER_PERF, CL_PERF,
-# etc. above.
-SUITES = {
-    CL_PERF: CommandLinePerformanceTest,
-    TIME_SIZE: CompileTimeAndSizeTest,
-    BROWSER_PERF: BrowserStandalonePerformanceTest,
-    DROMAEO: DromaeoTest,
-    DROMAEO_SIZE: DromaeoSizeTest,
-}
-
-def parse_args():
-  parser = optparse.OptionParser()
-  # TODO(vsm): Change to a list to scale.
-  parser.add_option('--suites', '-s', dest='suites',
-      help='Run the specified comma-separated test suites from set: %s' % \
-                      ','.join(SUITES.keys()),
-      action='store', default=None)
-  parser.add_option('--forever', '-f', dest='continuous',
-      help='Run this script forever, always checking for the next svn '
-      'checkin', action='store_true', default=False)
-  parser.add_option('--graph-only', '-g', dest='graph_only', default=False,
-      help='Do not run tests, only regenerate graphs', action='store_true')
-  parser.add_option('--nobuild', '-n', dest='no_build', action='store_true',
-      help='Do not sync with the repository and do not rebuild.', default=False)
-  parser.add_option('--upload', '-u', dest='upload',
-      help='Upload data to app engine (will require authentication).', 
-      action='store_true', default=False)
-  parser.add_option('--verbose', '-v', dest='verbose',
-      help='Print extra debug output', action='store_true', default=False)
-
-  args, ignored = parser.parse_args()
-
-  if not args.suites:
-    suites = SUITES.values()
-  else:
-    suites = []
-    suitelist = args.suites.split(',')
-    for name in suitelist:
-      if name in SUITES:
-        suites.append(SUITES[name])
-      else:
-        print 'Error: Invalid suite %s not in %s' % (name,
-                                                     ','.join(SUITES.keys()))
-        sys.exit(1)
-  return (suites, args.continuous, args.verbose, args.no_build,
-          args.graph_only, args.upload)
-
-def run_test_sequence(suites, no_build, graph_only, upload):
-  # The buildbot already builds and syncs to a specific revision. Don't fight
-  # with it or replicate work.
-  if not no_build and sync_and_build() == 1:
-    return # The build is broken.
-
-  for test in suites:
-    test().run(graph_only)
-
-  if upload:
-    upload_to_app_engine(SUITES.keys())
+          for metric in self.test.values_list:
+            if metric in line:
+              num = tokens[0]
+              if num.find('.') == -1:
+                num = int(num)
+              else:
+                num = float(num)
+              self.test.values_dict['commandline']['frog'][metric] += [num]
+              self.test.revision_dict['commandline']['frog'][metric] += \
+                  [revision_num]
+
+      if revision_num != 0:
+        for metric in self.test.values_list:
+          self.test.revision_dict['commandline']['frog'][metric].pop()
+          self.test.revision_dict['commandline']['frog'][metric] += \
+              [revision_num]
+          # Fill in 0 if compilation failed.
+          if self.test.values_dict['commandline']['frog'][metric][-1] < \
+              self.test.failure_threshold[metric]:
+            self.test.values_dict['commandline']['frog'][metric] += [0]
+            self.test.revision_dict['commandline']['frog'][metric] += \
+                [revision_num]
+
+      f.close()

[vsm, 2012/04/09 20:19:58]

Extra newline for new class.

[Emily Fortuna, 2012/04/09 21:15:03]

Done.

+  class CompileGrapher(Grapher):
+
+    def plot_results(self, png_filename):
+      self.style_and_save_perf_plot(
+          'Compiled minfrog Sizes', 'Size (in bytes)', 10, 10, 'lower left',
+          png_filename, ['commandline'], ['frog'], 
+          ['swarm', 'total', 'minfrog'])
+
+      self.style_and_save_perf_plot(
+          'Time to compile and bootstrap',
+          'Seconds', 10, 10, 'lower left', '2' + png_filename, ['commandline'],
+          ['frog'], ['Bootstrapping', 'Compiling on Dart VM'])
+
+
+class TestBuilder(object):
+  """Construct the desired test object."""
+  available_suites = {
+      BenchpressAndCoCommandLineTest.name(): BenchpressAndCoCommandLineTest,
+      CompileTimeAndSizeTest.name(): CompileTimeAndSizeTest,
+      BenchpressAndCoBrowserTest.name(): BenchpressAndCoBrowserTest,
+      DromaeoTest.name(): DromaeoTest,
+      DromaeoSizeTest.name(): DromaeoSizeTest,

[vsm, 2012/04/09 20:19:58]

Can you get rid of the redundancy here?  E.g., something like:

available_suites = dict((suite.name(), suite) for suite in [
  BenchpressAndCoCommandLineTest,
  CompileTimeAndSizeTest.
  ...
])

[Emily Fortuna, 2012/04/09 21:15:03]

Ah, much nicer. done.

+  }
+
+  def make_test(self, test_name, test_runner):

[vsm, 2012/04/09 20:19:58]

Why is this an instance method but not the below?

[Emily Fortuna, 2012/04/09 21:15:03]

Done.

+    return TestBuilder.available_suites[test_name](test_runner)
+
+  @staticmethod
+  def available_suite_names():
+    return TestBuilder.available_suites.keys()
+
 
 def main():
-  global VERBOSE
-  (suites, continuous, verbose, no_build, graph_only, upload) = parse_args()
-  VERBOSE = verbose
+  runner = TestRunner()
+  (suites, continuous, verbose, no_build, graph_only, upload) = \
+      runner.parse_args()

[vsm, 2012/04/09 20:19:58]

I suggest changing this method to only return continuous.  Every other value is
only passed right back to runner and just clutters this method.

[Emily Fortuna, 2012/04/09 21:15:03]

Done.

+  runner.verbose = verbose
   if continuous:
     while True:
-      if has_new_code():
-        run_test_sequence(suites, no_build, graph_only, upload)
+      if runner.has_new_code():
+        runner.run_test_sequence(suites, no_build, graph_only, upload)
       else:
-        time.sleep(SLEEP_TIME)
+        time.sleep(200)
   else:
-    run_test_sequence(suites, no_build, graph_only, upload)
+    runner.run_test_sequence(suites, no_build, graph_only, upload)
 
 if __name__ == '__main__':
   main()