Add Dartium to browser_perf tests

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import getpass
 import math
 try:
@@ skipping 323 matching lines @@
       self.values_dict[platform] = dict()
       for f in variants:
         self.revision_dict[platform][f] = dict()
         self.values_dict[platform][f] = dict()
         for val in values_list:
           self.revision_dict[platform][f][val] = []
           self.values_dict[platform][f][val] = []
         for extra_metric in extra_metrics:
           self.revision_dict[platform][f][extra_metric] = []
           self.values_dict[platform][f][extra_metric] = []
-  
+
+  def is_valid_combination(self, platform, variant):
+    """Check whether data should be captured for this platform/variant
+    combination.
+    """
+    return True
+
   def run(self, graph_only):
     """Run the benchmarks/tests from the command line and plot the
     results.
     
     Args:
       graph_only: True if we should just graph the results instead of also
           running tests."""
     for visitor in [self.tester, self.file_processor, self.grapher]:
       visitor.prepare()
     
@@ skipping 22 matching lines @@
   methods that many Tester objects use. Any class that would like to be a
   TesterVisitor must implement the run_tests() method."""
 
   def __init__(self, test):
     self.test = test
 
   def prepare(self):
     """Perform any initial setup required before the test is run."""
     pass
 
-  def add_svn_revision_to_trace(self, outfile):
+  def add_svn_revision_to_trace(self, outfile, browser = None):
     """Add the svn version number to the provided tracefile."""
     def search_for_revision(svn_info_command):
       p = subprocess.Popen(svn_info_command, stdout = subprocess.PIPE,
                            stderr = subprocess.STDOUT, shell =
                            self.test.test_runner.has_shell)
       output, _ = p.communicate()
       for line in output.split('\n'):
         if 'Revision' in line:
           self.test.test_runner.run_cmd(['echo', line.strip()], outfile)
           return True
       return False
 
-    if not search_for_revision(['svn', 'info']):
+    def get_dartium_revision():
+      version_file_name = os.path.join(DART_INSTALL_LOCATION, 'client', 'tests',
+                                       'dartium', 'LAST_VERSION')
+      version_file = open(version_file_name, 'r')
+      version = version_file.read().split('.')[-2]
+      version_file.close()
+      return version
+
+    if browser and browser == 'dartium':
+      revision = get_dartium_revision()
+      self.test.test_runner.run_cmd(['echo', 'Revision: ' + revision], outfile)
+    elif not search_for_revision(['svn', 'info']):
       if not search_for_revision(['git', 'svn', 'info']):
         self.test.test_runner.run_cmd(['echo', 'Revision: unknown'], outfile)
 
 
 class Processor(object):
   """The base level vistor class that processes tests. It contains convenience 
   methods that many File Processor objects use. Any class that would like to be
   a ProcessorVisitor must implement the process_file() method."""
 
   def __init__(self, test):
     self.test = test
 
   def prepare(self):
     """Perform any initial setup required before the test is run."""
     pass
 
   def calculate_geometric_mean(self, platform, variant, svn_revision):
     """Calculate the aggregate geometric mean for JS and frog benchmark sets,
     given two benchmark dictionaries."""
     geo_mean = 0
-    for benchmark in self.test.values_list:
-      geo_mean += math.log(self.test.values_dict[platform][variant][benchmark][
-          len(self.test.values_dict[platform][variant][benchmark]) - 1])
+    # TODO(vsm): Suppress graphing this combination altogether.  For
+    # now, we feed a geomean of 0.
+    if self.test.is_valid_combination(platform, variant):
+      for benchmark in self.test.values_list:
+        geo_mean += math.log(self.test.values_dict[platform][variant][benchmark][

[Emily Fortuna, 2012/04/17 19:59:29]

80 char.

[vsm, 2012/04/18 15:46:54]

Done.

+            len(self.test.values_dict[platform][variant][benchmark]) - 1])
 
     self.test.values_dict[platform][variant]['Geo-Mean'] += \
         [math.pow(math.e, geo_mean / len(self.test.values_list))]
     self.test.revision_dict[platform][variant]['Geo-Mean'] += [svn_revision]
 
 
 class Grapher(object):
   """The base level visitor class that generates graphs for data. It contains 
   convenience methods that many Grapher objects use. Any class that would like
   to be a GrapherVisitor must implement the plot_results() method."""
@@ skipping 123 matching lines @@
     def plot_avg_perf(self, png_filename):
       """Generate a plot that shows the performance changes of the geomentric
       mean of JS and frog benchmark performance over svn history."""
       (title, y_axis, size_x, size_y, loc, filename) = \
           ('Geometric Mean of benchmark %s performance on %s ' %
           (self.test.platform_type, utils.GuessOS()), 'Speed (bigger = better)',
           16, 5, 'lower left', 'avg'+png_filename)
       clear_axis = True
       for platform in self.test.platform_list:
         for version in self.test.versions:
-          for metric in self.test.extra_metrics:
-            self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc,
-                                          filename, [platform], [version],
-                                          [metric], clear_axis)
+          if self.test.is_valid_combination(platform, version):
+            for metric in self.test.extra_metrics:
+              self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc,
+                                            filename, [platform], [version],
+                                            [metric], clear_axis)
             clear_axis = False

[Emily Fortuna, 2012/04/17 19:59:29]

This line also needs to be indented two more spaces.

[vsm, 2012/04/18 15:46:54]

Thanks for the catch!

On 2012/04/17 19:59:29, Emily Fortuna wrote:

 
     def plot_results(self, png_filename):
       self.plot_all_perf(png_filename)
       self.plot_avg_perf('2' + png_filename)
 
 
 class CommonCommandLineTest(RuntimePerformanceTest):
   """Run the basic performance tests (Benchpress, some V8 benchmarks) from the
   command line."""
 
@@ skipping 65 matching lines @@
               [revision_num]
           self.test.values_dict['commandline']['frog'][benchmark] += \
               [frog_value]
       f.close()
 
       self.calculate_geometric_mean('commandline', 'frog', revision_num)
       self.calculate_geometric_mean('commandline', 'js', revision_num)
 
 
 class BrowserTester(Tester):
-  # TODO(vsm): Add Dartium.
   @staticmethod
   def get_browsers():
-    browsers = ['ff', 'chrome']
+    browsers = ['dartium', 'ff', 'chrome']
+    has_shell = False
     if platform.system() == 'Darwin':
       browsers += ['safari']
     if platform.system() == 'Windows':
       browsers += ['ie']
+      has_shell = True
+    if 'dartium' in browsers:
+      # Fetch it if necessary.
+      get_dartium = ['python',
+                     os.path.join(DART_INSTALL_LOCATION, 'tools', 'get_drt.py'),
+                     '--dartium']
+      # TODO(vsm): It's inconvenient that run_cmd isn't in scope here.

[Emily Fortuna, 2012/04/17 19:59:29]

Perhaps in any "BrowserTest" just before it is actually run, the Tester performs
any set up needed, such as downloading an updated version fo DRT. 

[vsm, 2012/04/18 15:46:54]

This does mean checking multiple times on a single run and potentially using
different versions for different Testers.

On 2012/04/17 19:59:29, Emily Fortuna wrote:

+      # Perhaps there is a better place to put that or this.
+      subprocess.Popen(get_dartium, shell=has_shell)
     return browsers
 
 
 class CommonBrowserTest(RuntimePerformanceTest):
   """Runs this basic performance tests (Benchpress, some V8 benchmarks) in the
   browser."""
 
   def __init__(self, test_runner):
     """Args:
       test_runner: Reference to the object that notifies us when to run."""
@@ skipping 18 matching lines @@
   class CommonBrowserTester(BrowserTester):
     def run_tests(self):
       """Run a performance test in the browser."""
       os.chdir('frog')
       self.test.test_runner.run_cmd(['python', os.path.join('benchmarks', 
                                      'make_web_benchmarks.py')])
       os.chdir('..')
 
       for browser in BrowserTester.get_browsers():
         for version in self.test.versions:
+          if not self.test.is_valid_combination(browser, version):
+            continue
           self.test.trace_file = os.path.join(
               'tools', 'testing', 'perf_testing', self.test.result_folder_name,
               'perf-%s-%s-%s' % (self.test.cur_time, browser, version))
-          self.add_svn_revision_to_trace(self.test.trace_file)
+          self.add_svn_revision_to_trace(self.test.trace_file, browser)
           file_path = os.path.join(
               os.getcwd(), 'internal', 'browserBenchmarks',
               'benchmark_page_%s.html' % version)
           self.test.test_runner.run_cmd(
               ['python', os.path.join('tools', 'testing', 'run_selenium.py'),
               '--out', file_path, '--browser', browser,
               '--timeout', '600', '--mode', 'perf'], self.test.trace_file, 
               append=True)
 
   class CommonBrowserFileProcessor(Processor):
@@ skipping 120 matching lines @@
         self.name(), BrowserTester.get_browsers(), 'browser',
         DromaeoTester.get_dromaeo_versions(), 
         DromaeoTester.get_dromaeo_benchmarks(), test_runner,
         self.DromaeoPerfTester(self),
         self.DromaeoFileProcessor(self))
 
   @staticmethod
   def name():
     return 'dromaeo'
 
+  def is_valid_combination(self, browser, version):
+    # TODO(vsm): This avoids a bug in 32-bit Chrome (dartium)
+    # running JS dromaeo.
+    if browser == 'dartium' and version == 'js':
+      return False
+    return True
+
   class DromaeoPerfTester(DromaeoTester):
     def run_tests(self):
       """Run dromaeo in the browser."""
 
       # Build tests.
       dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
       current_path = os.getcwd()
       os.chdir(dromaeo_path)
       self.test.test_runner.run_cmd(['python', 'generate_frog_tests.py'])
       os.chdir(current_path)
 
       versions = DromaeoTester.get_dromaeo_versions()
 
       for browser in BrowserTester.get_browsers():
         for version_name in versions:
+          if not self.test.is_valid_combination(browser, version):
+            continue
           version = DromaeoTest.DromaeoPerfTester.get_dromaeo_url_query(
-              version_name)
+              browser, version_name)
           self.test.trace_file = os.path.join(
               'tools', 'testing', 'perf_testing', self.test.result_folder_name,
               'dromaeo-%s-%s-%s' % (self.test.cur_time, browser, version_name))
-          self.add_svn_revision_to_trace(self.test.trace_file)
+          self.add_svn_revision_to_trace(self.test.trace_file, browser)
           file_path = os.path.join(os.getcwd(), dromaeo_path,
               'index-js.html?%s' % version)
           self.test.test_runner.run_cmd(
               ['python', os.path.join('tools', 'testing', 'run_selenium.py'),
                '--out', file_path, '--browser', browser,
                '--timeout', '600', '--mode', 'dromaeo'], self.test.trace_file,
                append=True)
 
     @staticmethod
-    def get_dromaeo_url_query(version):
+    def get_dromaeo_url_query(browser, version):
+      if browser == 'dartium':
+        version = version.replace('frog', 'dart')
       version = version.replace('_','&')
       tags = DromaeoTester.get_valid_dromaeo_tags()
       return '|'.join([ '%s&%s' % (version, tag) for tag in tags])
 
 
   class DromaeoFileProcessor(Processor):
     def process_file(self, afile):
       """Comb through the html to find the performance results."""
       parts = afile.split('-')
       browser = parts[2]
@@ skipping 331 matching lines @@
     while True:
       if runner.has_new_code():
         runner.run_test_sequence()
       else:
         time.sleep(200)
   else:
     runner.run_test_sequence()
 
 if __name__ == '__main__':
   main()