Don't upload data to appengine.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
-import getpass
-import math
-try:
-  from matplotlib.font_manager import FontProperties
-  import matplotlib.pyplot as plt
-except ImportError:
-  pass # Only needed if we want to make graphs.
 import optparse
 import os
 from os.path import dirname, abspath
 import platform
 import re
 import shutil
 import stat
 import subprocess
 import sys
 import time
-import traceback
 
 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__)))))
 DART_INSTALL_LOCATION = abspath(os.path.join(dirname(abspath(__file__)),
                                              '..', '..', '..'))
 sys.path.append(TOOLS_PATH)
+sys.path.append(os.path.join(DART_INSTALL_LOCATION, 'internal', 'tests'))
+import post_results
 import utils
 
 """This script runs to track performance and size progress of
 different svn revisions. It tests to see if there a newer version of the code on
 the server, and will sync and run the performance tests if so."""
 class TestRunner(object):
 
   def __init__(self):
     self.verbose = False
     self.has_shell = False
@@ skipping 67 matching lines @@
 
     self.run_cmd(['gclient', 'sync'])
 
     # On Windows, the output directory is marked as "Read Only," which causes an
     # error to be thrown when we use shutil.rmtree. This helper function changes
     # the permissions so we can still delete the directory.
     def on_rm_error(func, path, exc_info):
       if os.path.exists(path):
         os.chmod(path, stat.S_IWRITE)
         os.unlink(path)
-    # TODO(efortuna): building the sdk locally is a band-aid until all build
+    # TODO(efortuna): building the sdk locally is a band-aid until all build XXX
     # platform SDKs are hosted in Google storage. Pull from https://sandbox.
     # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk
     # eventually.
     # TODO(efortuna): Currently always building ia32 architecture because we 
     # don't have test statistics for what's passing on x64. Eliminate arch 
     # specification when we have tests running on x64, too.
     shutil.rmtree(os.path.join(os.getcwd(),
                   utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32')),
                   onerror=on_rm_error)
 
@@ skipping 35 matching lines @@
   def get_os_directory(self):
     """Specifies the name of the directory for the testing build of dart, which
     has yet a different naming convention from utils.getBuildRoot(...)."""
     if platform.system() == 'Windows':
       return 'windows'
     elif platform.system() == 'Darwin':
       return 'macos'
     else:
       return 'linux'
 
-  def upload_to_app_engine(self, suite_names):
-    """Upload our results to our appengine server.
-    Arguments:
-      suite_names: Directories to upload data from (should match directory 
-        names)."""
-    # TODO(efortuna): This is the most basic way to get the data up
-    # for others to view. Revisit this once we're serving nicer graphs (Google
-    # Chart Tools) and from multiple perfbots and once we're in a position to
-    # organize the data in a useful manner(!!).
-    os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
-                          'perf_testing'))
-    for data in suite_names:
-      path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS())
-      shutil.rmtree(path, ignore_errors=True)
-      os.makedirs(path)
-      files = []
-      # Copy the 1000 most recent trace files to be uploaded.
-      for f in os.listdir(data):
-        files += [(os.path.getmtime(os.path.join(data, f)), f)]
-      files.sort()
-      for f in files[-1000:]:
-        shutil.copyfile(os.path.join(data, f[1]), 
-                        os.path.join(path, f[1]+'.txt'))
-    # Generate directory listing.
-    for data in suite_names:
-      path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS())
-      out = open(os.path.join('appengine', 'static',
-                 '%s-%s.html' % (data, utils.GuessOS())), 'w')
-      out.write('<html>\n  <body>\n    <ul>\n')
-      for f in os.listdir(path):
-        if not f.startswith('.'):
-          out.write('      <li><a href=data' + \
-                    '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \
-                    {'data': data, 'os': utils.GuessOS(), 'file': f})
-      out.write('    </ul>\n  </body>\n</html>')
-      out.close()
-
-    shutil.rmtree(os.path.join('appengine', 'static', 'graphs'),
-                  ignore_errors=True)
-    shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs'))
-    shutil.copyfile('index.html', os.path.join('appengine', 'static',
-                    'index.html'))
-    shutil.copyfile('dromaeo.html', os.path.join('appengine', 'static',
-                    'dromaeo.html'))
-    shutil.copyfile('data.html', os.path.join('appengine', 'static',
-                    'data.html'))
-    self.run_cmd([os.path.join('..', '..', '..', 'third_party',
-                               'appengine-python', 'appcfg.py'), '--oauth2', 
-                               'update', 'appengine/'])
-
   def parse_args(self):
     parser = optparse.OptionParser()
     parser.add_option('--suites', '-s', dest='suites', help='Run the specified '
                       'comma-separated test suites from set: %s' % \
                       ','.join(TestBuilder.available_suite_names()), 
                       action='store', default=None)
     parser.add_option('--forever', '-f', dest='continuous', help='Run this scri'
                       'pt forever, always checking for the next svn checkin',
                       action='store_true', default=False)
-    parser.add_option('--graph-only', '-g', dest='graph_only', default=False,
-                      help='Do not run tests, only regenerate graphs', 
-                      action='store_true')
     parser.add_option('--nobuild', '-n', dest='no_build', action='store_true',
                       help='Do not sync with the repository and do not '
                       'rebuild.', default=False)
-    parser.add_option('--upload', '-u', dest='upload', help='Upload data to '
-                      'app engine (will require authentication).', 
-                      action='store_true', default=False)
     parser.add_option('--verbose', '-v', dest='verbose', help='Print extra '
                       'debug output', action='store_true', default=False)
 
     args, ignored = parser.parse_args()
 
     if not args.suites:
       suites = TestBuilder.available_suite_names()
     else:
       suites = []
       suitelist = args.suites.split(',')
       for name in suitelist:
         if name in TestBuilder.available_suite_names():
           suites.append(name)
         else:
           print ('Error: Invalid suite %s not in ' % name) + \
               '%s' % ','.join(TestBuilder.available_suite_names())
           sys.exit(1)
     self.suite_names = suites
     self.no_build = args.no_build
-    self.graph_only = args.graph_only
-    self.upload = args.upload
     self.verbose = args.verbose
     return args.continuous
 
   def run_test_sequence(self):
     """Run the set of commands to (possibly) build, run, and graph the results
     of our tests.
 
     Args:
       suite_names: The "display name" the user enters to specify which
           benchmark(s) to run.
       no_build: True if we should not check the repository and build the latest
           version.
-      graph_only: True if we should not run the tests, just (re)generate graphs.
-      upload: True if we should upload our results to appengine."""
+    """
     suites = []
     for name in self.suite_names:
       suites += [TestBuilder.make_test(name, self)]
 
     if not self.no_build and self.sync_and_build(suites) == 1:
       return # The build is broken.
 
     for test in suites:
-      test.run(self.graph_only)
-
-    if self.upload:
-      self.upload_to_app_engine(TestBuilder.available_suite_names())
+      test.run()
 
 
 class Test(object):
   """The base class to provide shared code for different tests we will run and
   graph. At a high level, each test has three visitors (the tester, the
-  file_processor, and the grapher) that perform operations on the test 
-  object."""
+  file_processor that perform operations on the test object."""
 
   def __init__(self, result_folder_name, platform_list, variants,
-               values_list, test_runner, tester, file_processor, grapher,
-               extra_metrics=['Geo-Mean'], build_targets=['create_sdk']):
+               values_list, test_runner, tester, file_processor,
+               build_targets=['create_sdk']):
     """Args:
          result_folder_name: The name of the folder where a tracefile of
              performance results will be stored.
          platform_list: A list containing the platform(s) that our data has been
              run on. (command line, firefox, chrome, etc)
          variants: A list specifying whether we hold data about Frog
              generated code, plain JS code, or a combination of both, or
              Dart depending on the test.
          values_list: A list containing the type of data we will be graphing
              (benchmarks, percentage passing, etc).
          test_runner: Reference to the parent test runner object that notifies a
              test when to run.
          tester: The visitor that actually performs the test running mechanics.
          file_processor: The visitor that processes files in the format
              appropriate for this test.
-         grapher: The visitor that generates graphs given our test result data.
-         extra_metrics: A list of any additional measurements we wish to keep
-             track of (such as the geometric mean of a set, the sum, etc).
          build_targets: The targets necessary to build to run these tests
              (default target is create_sdk)."""
     self.result_folder_name = result_folder_name
     # cur_time is used as a timestamp of when this performance test was run.
     self.cur_time = str(time.mktime(datetime.datetime.now().timetuple()))
     self.values_list = values_list
     self.platform_list = platform_list
-    self.revision_dict = dict()
-    self.values_dict = dict()
     self.test_runner = test_runner
     self.tester = tester
     self.file_processor = file_processor
-    self.grapher = grapher
-    self.extra_metrics = extra_metrics
     self.build_targets = build_targets
-    # Initialize our values store.
-    for platform in platform_list:
-      self.revision_dict[platform] = dict()
-      self.values_dict[platform] = dict()
-      for f in variants:
-        self.revision_dict[platform][f] = dict()
-        self.values_dict[platform][f] = dict()
-        for val in values_list:
-          self.revision_dict[platform][f][val] = []
-          self.values_dict[platform][f][val] = []
-        for extra_metric in extra_metrics:
-          self.revision_dict[platform][f][extra_metric] = []
-          self.values_dict[platform][f][extra_metric] = []
 
   def is_valid_combination(self, platform, variant):
     """Check whether data should be captured for this platform/variant
     combination.
     """
     return True
 
-  def run(self, graph_only):
+  def run(self):
     """Run the benchmarks/tests from the command line and plot the
     results.
-    
-    Args:
-      graph_only: True if we should just graph the results instead of also
-          running tests."""
-    for visitor in [self.tester, self.file_processor, self.grapher]:
+    """
+    for visitor in [self.tester, self.file_processor]:
       visitor.prepare()
     
     os.chdir(DART_INSTALL_LOCATION)
     self.test_runner.ensure_output_directory(self.result_folder_name)
-    if not graph_only:
-      self.tester.run_tests()
+    self.tester.run_tests()
 
     os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
 
-    # TODO(efortuna): You will want to make this only use a subset of the files
-    # eventually.
+    # TODO(efortuna): Remove trace files once uploaded. This will happen in a
+    # future CL.
     files = os.listdir(self.result_folder_name)
-
     for afile in files:
       if not afile.startswith('.'):
         self.file_processor.process_file(afile)
 
-    if 'plt' in globals():
-      # Only run Matplotlib if it is installed.
-      self.grapher.plot_results('%s.png' % self.result_folder_name)
-
 
 class Tester(object):
   """The base level visitor class that runs tests. It contains convenience 
   methods that many Tester objects use. Any class that would like to be a
   TesterVisitor must implement the run_tests() method."""
 
   def __init__(self, test):
     self.test = test
 
   def prepare(self):
@@ skipping 34 matching lines @@
   methods that many File Processor objects use. Any class that would like to be
   a ProcessorVisitor must implement the process_file() method."""
 
   def __init__(self, test):
     self.test = test
 
   def prepare(self):
     """Perform any initial setup required before the test is run."""
     pass
 
-  def calculate_geometric_mean(self, platform, variant, svn_revision):
-    """Calculate the aggregate geometric mean for JS and frog benchmark sets,
-    given two benchmark dictionaries."""
-    geo_mean = 0
-    # TODO(vsm): Suppress graphing this combination altogether.  For
-    # now, we feed a geomean of 0.
-    if self.test.is_valid_combination(platform, variant):
-      for benchmark in self.test.values_list:
-        geo_mean += math.log(
-            self.test.values_dict[platform][variant][benchmark][
-                len(self.test.values_dict[platform][variant][benchmark]) - 1])
-
-    self.test.values_dict[platform][variant]['Geo-Mean'] += \
-        [math.pow(math.e, geo_mean / len(self.test.values_list))]
-    self.test.revision_dict[platform][variant]['Geo-Mean'] += [svn_revision]
-
-
-class Grapher(object):
-  """The base level visitor class that generates graphs for data. It contains 
-  convenience methods that many Grapher objects use. Any class that would like
-  to be a GrapherVisitor must implement the plot_results() method."""
+  def report_results(self, benchmark_name, score, platform, variant, 
+                     revision_number):
+    """Store the results of the benchmark run.
+    Args:
+      benchmark_name: The name of the individual benchmark.
+      score: The numerical value of this benchmark.
+      platform: The platform the test was run on (firefox, command line, etc).
+      variant: Specifies whether the data was about generated Frog, js, a
+          combination of both, or Dart depending on the test.
+      revision_number: The revision of the code (and sometimes the revision of
+          dartium).
   
-  graph_out_dir = 'graphs'
-  
-  def __init__(self, test):
-    self.color_index = 0
-    self.test = test
-  
-  def prepare(self):
-    """Perform any initial setup required before the test is run."""
-    if 'plt' in globals():
-      plt.cla() # cla = clear current axes
-    else:
-      print 'Unable to import Matplotlib and therefore unable to generate ' + \
-          'graphs. Please install it for this version of Python.'
-    self.test.test_runner.ensure_output_directory(Grapher.graph_out_dir)
-
-  def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y,
-                               legend_loc, filename, platform_list, variants, 
-                               values_list, should_clear_axes=True):
-    """Sets style preferences for chart boilerplate that is consistent across
-    all charts, and saves the chart as a png.
-
-    Args:
-      size_x: the size of the printed chart, in inches, in the horizontal
-        direction
-      size_y: the size of the printed chart, in inches in the vertical direction
-      legend_loc: the location of the legend in on the chart. See suitable
-        arguments for the loc argument in matplotlib
-      filename: the filename that we want to save the resulting chart as
-      platform_list: a list containing the platform(s) that our data has been
-        run on. (command line, firefox, chrome, etc)
-      values_list: a list containing the type of data we will be graphing
-        (performance, percentage passing, etc)
-      should_clear_axes: True if we want to create a fresh graph, instead of
-        plotting additional lines on the current graph."""
-    if should_clear_axes:
-      plt.cla() # cla = clear current axes
-    for platform in platform_list:
-      for f in variants:
-        for val in values_list:
-          plt.plot(self.test.revision_dict[platform][f][val],
-              self.test.values_dict[platform][f][val],
-              color=self.get_color(), label='%s-%s-%s' % (platform, f, val))
-
-    plt.xlabel('Revision Number')
-    plt.ylabel(y_axis_label)
-    plt.title(chart_title)
-    fontP = FontProperties()
-    fontP.set_size('small')
-    plt.legend(loc=legend_loc, prop = fontP)
-
-    fig = plt.gcf()
-    fig.set_size_inches(size_x, size_y)
-    fig.savefig(os.path.join(Grapher.graph_out_dir, filename))
-  
-  def get_color(self):
-    # Just a bunch of distinct colors for a potentially large number of values
-    # we wish to graph.
-    colors = [
-        'blue', 'green', 'red', 'cyan', 'magenta', 'black', '#3366CC',
-        '#DC3912', '#FF9900', '#109618', '#990099', '#0099C6', '#DD4477',
-        '#66AA00', '#B82E2E', '#316395', '#994499', '#22AA99', '#AAAA11',
-        '#6633CC', '#E67300', '#8B0707', '#651067', '#329262', '#5574A6',
-        '#3B3EAC', '#B77322', '#16D620', '#B91383', '#F4359E', '#9C5935',
-        '#A9C413', '#2A778D', '#668D1C', '#BEA413', '#0C5922', '#743411',
-        '#45AFE2', '#FF3300', '#FFCC00', '#14C21D', '#DF51FD', '#15CBFF',
-        '#FF97D2', '#97FB00', '#DB6651', '#518BC6', '#BD6CBD', '#35D7C2',
-        '#E9E91F', '#9877DD', '#FF8F20', '#D20B0B', '#B61DBA', '#40BD7E',
-        '#6AA7C4', '#6D70CD', '#DA9136', '#2DEA36', '#E81EA6', '#F558AE',
-        '#C07145', '#D7EE53', '#3EA7C6', '#97D129', '#E9CA1D', '#149638',
-        '#C5571D']
-    color = colors[self.color_index]
-    self.color_index = (self.color_index + 1) % len(colors)
-    return color
+    Returns: True if the post was successful."""
+    return post_results.report_results(benchmark_name, score, platform, variant,
+                                revision_number)
 
 
 class RuntimePerformanceTest(Test):
   """Super class for all runtime performance testing."""
 
   def __init__(self, result_folder_name, platform_list, platform_type,
                versions, benchmarks, test_runner, tester, file_processor,
                build_targets=['create_sdk']):
     """Args:
         result_folder_name: The name of the folder where a tracefile of
             performance results will be stored.
         platform_list: A list containing the platform(s) that our data has been
             run on. (command line, firefox, chrome, etc)
         variants: A list specifying whether we hold data about Frog
             generated code, plain JS code, or a combination of both, or
             Dart depending on the test.
         values_list: A list containing the type of data we will be graphing
             (benchmarks, percentage passing, etc).
         test_runner: Reference to the parent test runner object that notifies a
             test when to run.
         tester: The visitor that actually performs the test running mechanics.
         file_processor: The visitor that processes files in the format
             appropriate for this test.
-        grapher: The visitor that generates graphs given our test result data.
-        extra_metrics: A list of any additional measurements we wish to keep
-            track of (such as the geometric mean of a set, the sum, etc).
         build_targets: The targets necessary to build to run these tests
             (default target is create_sdk)."""
     super(RuntimePerformanceTest, self).__init__(result_folder_name,
           platform_list, versions, benchmarks, test_runner, tester,
-          file_processor, self.RuntimePerfGrapher(self),
-          build_targets=build_targets)
+          file_processor, build_targets=build_targets)
     self.platform_list = platform_list
     self.platform_type = platform_type
     self.versions = versions
     self.benchmarks = benchmarks 
  
-  class RuntimePerfGrapher(Grapher):
-    def plot_all_perf(self, png_filename):
-      """Create a plot that shows the performance changes of individual
-      benchmarks run by JS and generated by frog, over svn history."""
-      for benchmark in self.test.benchmarks:
-        self.style_and_save_perf_plot(
-            'Performance of %s over time on the %s on %s' % (benchmark,
-            self.test.platform_type, utils.GuessOS()), 
-            'Speed (bigger = better)', 16, 14, 'lower left',
-            benchmark + png_filename, self.test.platform_list,
-            self.test.versions, [benchmark])
-
-    def plot_avg_perf(self, png_filename):
-      """Generate a plot that shows the performance changes of the geomentric
-      mean of JS and frog benchmark performance over svn history."""
-      (title, y_axis, size_x, size_y, loc, filename) = \
-          ('Geometric Mean of benchmark %s performance on %s ' %
-          (self.test.platform_type, utils.GuessOS()), 'Speed (bigger = better)',
-          16, 5, 'lower left', 'avg'+png_filename)
-      clear_axis = True
-      for platform in self.test.platform_list:
-        for version in self.test.versions:
-          if self.test.is_valid_combination(platform, version):
-            for metric in self.test.extra_metrics:
-              self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc,
-                                            filename, [platform], [version],
-                                            [metric], clear_axis)
-              clear_axis = False
-
-    def plot_results(self, png_filename):
-      self.plot_all_perf(png_filename)
-      self.plot_avg_perf('2' + png_filename)
-
-
-class CommonCommandLineTest(RuntimePerformanceTest):
-  """Run the basic performance tests (Benchpress, some V8 benchmarks) from the
-  command line."""
-
-  def __init__(self, test_runner):
-    """Args:
-      test_runner: Reference to the object that notfies this test when to
-          run."""
-    super(CommonCommandLineTest, self).__init__(
-          self.name(), ['commandline'], 
-          'command line', ['js', 'frog'], self.get_standalone_benchmarks(), 
-          test_runner, self.CommonCommandLineTester(self),
-          self.CommonCommandLineFileProcessor(self),
-          build_targets=['create_sdk', 'dart2js'])
-
-  @staticmethod
-  def name():
-    return 'cl-perf'
-
-  @staticmethod
-  def get_standalone_benchmarks():
-    return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees',
-    'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute',
-    'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers',
-    'TreeSort']
-
-  class CommonCommandLineTester(Tester):
-    def run_tests(self):
-      """Run a performance test on our updated system."""
-      os.chdir('frog')
-      self.test.trace_file = os.path.join(
-          '..', 'tools', 'testing', 'perf_testing', 
-          self.test.result_folder_name, 'result' + self.test.cur_time)
-      self.test.test_runner.run_cmd(['python', os.path.join('benchmarks', 
-                                     'perf_tests.py')], self.test.trace_file)
-      os.chdir('..')
-
-  class CommonCommandLineFileProcessor(Processor):
-    def process_file(self, afile):
-      """Pull all the relevant information out of a given tracefile.
-
-      Args:
-        afile: The filename string we will be processing."""
-      os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
-          'testing', 'perf_testing'))
-      f = open(os.path.join(self.test.result_folder_name, afile))
-      tabulate_data = False
-      revision_num = 0
-      for line in f.readlines():
-        if 'Revision' in line:
-          revision_num = int(line.split()[1])
-        elif 'Benchmark' in line:
-          tabulate_data = True
-        elif tabulate_data:
-          tokens = line.split()
-          if len(tokens) < 4 or tokens[0] not in self.test.benchmarks:
-            #Done tabulating data.
-            break
-          js_value = float(tokens[1])
-          frog_value = float(tokens[3])
-          if js_value == 0 or frog_value == 0:
-            #Then there was an error when this performance test was run. Do not
-            #count it in our numbers.
-            return
-          benchmark = tokens[0]
-          self.test.revision_dict['commandline']['js'][benchmark] += \
-              [revision_num]
-          self.test.values_dict['commandline']['js'][benchmark] += [js_value]
-          self.test.revision_dict['commandline']['frog'][benchmark] += \
-              [revision_num]
-          self.test.values_dict['commandline']['frog'][benchmark] += \
-              [frog_value]
-      f.close()
-
-      self.calculate_geometric_mean('commandline', 'frog', revision_num)
-      self.calculate_geometric_mean('commandline', 'js', revision_num)
-
 
 class BrowserTester(Tester):
   @staticmethod
   def get_browsers():
     browsers = ['dartium', 'ff', 'chrome']
     has_shell = False
     if platform.system() == 'Darwin':
       browsers += ['safari']
     if platform.system() == 'Windows':
       browsers += ['ie']
@@ skipping 90 matching lines @@
       else:
         results = line.split('<br />')
       for result in results:
         name_and_score = result.split(':')
         if len(name_and_score) < 2:
           break
         name = name_and_score[0].strip()
         score = name_and_score[1].strip()
         if version == 'js' or version == 'v8':
           version = 'js'
-          bench_dict = self.test.values_dict[browser]['js']
-        else:
-          bench_dict = self.test.values_dict[browser]['frog']
-        bench_dict[name] += [float(score)]
-        self.test.revision_dict[browser][version][name] += [revision_num]
+        self.report_results(name, score, browser, version, revision_num)
 
       f.close()
-      self.calculate_geometric_mean(browser, version, revision_num)
+
 
 class DromaeoTester(Tester):
   DROMAEO_BENCHMARKS = {
       'attr': ('attributes', [
           'getAttribute',
           'element.property',
           'setAttribute',
           'element.property = value']),
       'modify': ('modify', [
           'createElement',
@@ skipping 13 matching lines @@
           'getElementsByName',
           'getElementsByName (not in document)']),
       'traverse': ('traverse', [
           'firstChild',
           'lastChild',
           'nextSibling',
           'previousSibling',
           'childNodes'])
   }
 
-  # Use legal appengine filenames for benchmark names.
-  @staticmethod
-  def legalize_filename(str):
-    remap = {
-        ' ': '_',
-        '(': '_',
-        ')': '_',
-        '*': 'ALL',
-        '=': 'ASSIGN',
-        }
-    for (old, new) in remap.iteritems():
-      str = str.replace(old, new)
+  # Use filenames that don't have unusual characters for benchmark names.       
+  @staticmethod 
+  def legalize_filename(str):   
+    remap = {   
+        ' ': '_',       
+        '(': '_',       
+        ')': '_',       
+        '*': 'ALL',     
+        '=': 'ASSIGN',  
+        }       
+    for (old, new) in remap.iteritems():        
+      str = str.replace(old, new)       
     return str
 
   # TODO(vsm): This is a hack to skip breaking tests.  Triage this
   # failure properly.  The modify suite fails on 32-bit chrome on
   # the mac.
   @staticmethod
   def get_valid_dromaeo_tags():
     tags = [tag for (tag, _) in DromaeoTester.DROMAEO_BENCHMARKS.values()]
     if platform.system() == 'Darwin':
       tags.remove('modify')
@@ skipping 42 matching lines @@
       dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
       current_path = os.getcwd()
       os.chdir(dromaeo_path)
       self.test.test_runner.run_cmd(['python', 'generate_frog_tests.py'])
       os.chdir(current_path)
 
       versions = DromaeoTester.get_dromaeo_versions()
 
       for browser in BrowserTester.get_browsers():
         for version_name in versions:
-          if not self.test.is_valid_combination(browser, version):
+          if not self.test.is_valid_combination(browser, version_name):
             continue
           version = DromaeoTest.DromaeoPerfTester.get_dromaeo_url_query(
               browser, version_name)
           self.test.trace_file = os.path.join(
               'tools', 'testing', 'perf_testing', self.test.result_folder_name,
               'dromaeo-%s-%s-%s' % (self.test.cur_time, browser, version_name))
           self.add_svn_revision_to_trace(self.test.trace_file, browser)
           file_path = '"%s"' % os.path.join(os.getcwd(), dromaeo_path,
               'index-js.html?%s' % version)
           self.test.test_runner.run_cmd(
@@ skipping 11 matching lines @@
       return '|'.join([ '%s&%s' % (version, tag) for tag in tags])
 
 
   class DromaeoFileProcessor(Processor):
     def process_file(self, afile):
       """Comb through the html to find the performance results."""
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
 
-      bench_dict = self.test.values_dict[browser][version]
-
       f = open(os.path.join(self.test.result_folder_name, afile))
       lines = f.readlines()
       i = 0
       revision_num = 0
       revision_pattern = r'Revision: (\d+)'
       suite_pattern = r'<div class="result-item done">(.+?)</ol></div>'
       result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)'
 
       for line in lines:
         rev = re.match(revision_pattern, line.strip())
         if rev:
           revision_num = int(rev.group(1))
           continue
 
         suite_results = re.findall(suite_pattern, line)
         if suite_results:
           for suite_result in suite_results:
             results = re.findall(r'<li>(.*?)</li>', suite_result)
             if results:
               for result in results:
                 r = re.match(result_pattern, result)
-                name = DromaeoTester.legalize_filename(
-                    r.group(1).strip(':'))
+                name = DromaeoTester.legalize_filename(r.group(1).strip(':'))
                 score = float(r.group(2))
-                bench_dict[name] += [float(score)]
-                self.test.revision_dict[browser][version][name] += \
-                    [revision_num]
+                self.report_results(name, score, browser, version, revision_num)
 
       f.close()
-      self.calculate_geometric_mean(browser, version, revision_num)
 
 
 class DromaeoSizeTest(Test):
   """Run tests to determine the compiled file output size of Dromaeo."""
   def __init__(self, test_runner):
     super(DromaeoSizeTest, self).__init__(
         self.name(),
         ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
         DromaeoTester.DROMAEO_BENCHMARKS.keys(), test_runner, 
         self.DromaeoSizeTester(self),
-        self.DromaeoSizeProcessor(self),
-        self.DromaeoSizeGrapher(self), extra_metrics=['sum'])
+        self.DromaeoSizeProcessor(self))
   
   @staticmethod
   def name():
     return 'dromaeo-size'
 
 
   class DromaeoSizeTester(DromaeoTester):
     def run_tests(self):
       # Build tests.
       dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
@@ skipping 40 matching lines @@
             js_size = os.path.getsize(os.path.join(frog_path, name))
           except OSError:
             pass #If compilation failed, continue on running other tests.
 
           total_size[variant] += js_size
           self.test.test_runner.run_cmd(
               ['echo', 'Size (%s, %s): %s' % (variant, suite, str(js_size))],
               self.test.trace_file, append=True)
 
       self.test.test_runner.run_cmd(
-          ['echo', 'Size (dart, %s): %s' % (total_dart_size, 
-                                            self.test.extra_metrics[0])],
+          ['echo', 'Size (dart, %s): %s' % (total_dart_size, 'sum')],
           self.test.trace_file, append=True)
       for (variant, _) in variants:
         self.test.test_runner.run_cmd(
-            ['echo', 'Size (%s, %s): %s' % (variant, self.test.extra_metrics[0],
+            ['echo', 'Size (%s, %s): %s' % (variant, 'sum',
                                             total_size[variant])],
             self.test.trace_file, append=True)
 
+
   class DromaeoSizeProcessor(Processor):
     def process_file(self, afile):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
         afile: is the filename string we will be processing."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
       f = open(os.path.join(self.test.result_folder_name, afile))
       tabulate_data = False
       revision_num = 0
       revision_pattern = r'Revision: (\d+)'
       result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)'
 
       for line in f.readlines():
         rev = re.match(revision_pattern, line.strip())
         if rev:
           revision_num = int(rev.group(1))
           continue
 
         result = re.match(result_pattern, line.strip())
         if result:
           variant = result.group(1)
           metric = result.group(2)
           num = result.group(3)
           if num.find('.') == -1:
             num = int(num)
           else:
             num = float(num)
-          self.test.values_dict['browser'][variant][metric] += [num]
-          self.test.revision_dict['browser'][variant][metric] += [revision_num]
+          self.report_results(metric, num, 'browser', variant, revision_num)
 
       f.close()
-  class DromaeoSizeGrapher(Grapher):
-    def plot_results(self, png_filename):
-      self.style_and_save_perf_plot(
-          'Compiled Dromaeo Sizes',
-          'Size (in bytes)', 10, 10, 'lower left', png_filename,
-          ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
-          DromaeoTester.DROMAEO_BENCHMARKS.keys())
-
-      self.style_and_save_perf_plot(
-          'Compiled Dromaeo Sizes',
-          'Size (in bytes)', 10, 10, 'lower left', '2' + png_filename,
-          ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
-          [self.test.extra_metrics[0]])
   
 
 class CompileTimeAndSizeTest(Test):
   """Run tests to determine how long minfrog takes to compile, and the compiled
   file output size of some benchmarking files."""
   def __init__(self, test_runner):
     """Reference to the test_runner object that notifies us when to begin
     testing."""
     super(CompileTimeAndSizeTest, self).__init__(
         self.name(), ['commandline'], ['frog'], 
         ['Compiling on Dart VM', 'Bootstrapping', 'minfrog', 'swarm', 'total'],
         test_runner, self.CompileTester(self),
-        self.CompileProcessor(self), self.CompileGrapher(self))
+        self.CompileProcessor(self))
     self.dart_compiler = os.path.join(
         DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(),
         'release', 'ia32'), 'dart-sdk', 'bin', 'frogc')
     _suffix = ''
     if platform.system() == 'Windows':
       _suffix = '.exe'
     self.dart_vm = os.path.join(
         DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), 
         'release', 'ia32'), 'dart-sdk', 'bin','dart' + _suffix)
     self.failure_threshold = {
@@ skipping 59 matching lines @@
       self.test.test_runner.run_cmd(
           ['echo', '%d Generated checked swarm size' % swarm_size],
           self.test.trace_file, append=True)
 
       self.test.test_runner.run_cmd(
           ['echo', '%d Generated checked total size' % total_size],
           self.test.trace_file, append=True)
     
       #Revert our newly built minfrog to prevent conflicts when we update
       self.test.test_runner.run_cmd(
-          ['svn', 'revert',  os.path.join(os.getcwd(), 'frog', 'minfrog')])
-    
+          ['svn', 'revert',  os.path.join(os.getcwd(), 'minfrog')])
       os.chdir('..')
 
+
   class CompileProcessor(Processor):
+
     def process_file(self, afile):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
         afile: is the filename string we will be processing."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
       f = open(os.path.join(self.test.result_folder_name, afile))
       tabulate_data = False
       revision_num = 0
       for line in f.readlines():
         tokens = line.split()
         if 'Revision' in line:
           revision_num = int(line.split()[1])
         else:
           for metric in self.test.values_list:
             if metric in line:
               num = tokens[0]
               if num.find('.') == -1:
                 num = int(num)
               else:
                 num = float(num)
-              self.test.values_dict['commandline']['frog'][metric] += [num]
-              self.test.revision_dict['commandline']['frog'][metric] += \
-                  [revision_num]
-
-      if revision_num != 0:
-        for metric in self.test.values_list:
-          self.test.revision_dict['commandline']['frog'][metric].pop()
-          self.test.revision_dict['commandline']['frog'][metric] += \
-              [revision_num]
-          # Fill in 0 if compilation failed.
-          if self.test.values_dict['commandline']['frog'][metric][-1] < \
-              self.test.failure_threshold[metric]:
-            self.test.values_dict['commandline']['frog'][metric] += [0]
-            self.test.revision_dict['commandline']['frog'][metric] += \
-                [revision_num]
+              self.report_results(metric, num, 'commandline', 'frog', 
+                                 revision_num)
 
       f.close()
 
-  class CompileGrapher(Grapher):
-
-    def plot_results(self, png_filename):
-      self.style_and_save_perf_plot(
-          'Compiled minfrog Sizes', 'Size (in bytes)', 10, 10, 'lower left',
-          png_filename, ['commandline'], ['frog'], 
-          ['swarm', 'total', 'minfrog'])
-
-      self.style_and_save_perf_plot(
-          'Time to compile and bootstrap',
-          'Seconds', 10, 10, 'lower left', '2' + png_filename, ['commandline'],
-          ['frog'], ['Bootstrapping', 'Compiling on Dart VM'])
-
 
 class TestBuilder(object):
   """Construct the desired test object."""
   available_suites = dict((suite.name(), suite) for suite in [
-      CommonCommandLineTest, CompileTimeAndSizeTest,
-      CommonBrowserTest, DromaeoTest, DromaeoSizeTest])
+      CompileTimeAndSizeTest, CommonBrowserTest, DromaeoTest, DromaeoSizeTest])
 
   @staticmethod
   def make_test(test_name, test_runner):
     return TestBuilder.available_suites[test_name](test_runner)
 
   @staticmethod
   def available_suite_names():
     return TestBuilder.available_suites.keys()
 
 
 def main():
   runner = TestRunner()
   continuous = runner.parse_args()
   if continuous:
     while True:
       if runner.has_new_code():
         runner.run_test_sequence()
       else:
         time.sleep(200)
   else:
     runner.run_test_sequence()
 
 if __name__ == '__main__':
   main()