Daily update script for performance numbers

tools/testing/perf_testing/create_graph.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import math
 from matplotlib.font_manager import FontProperties
 import matplotlib.pyplot as plt
@@ skipping 19 matching lines @@
     '..', '..', '..')
 V8_MEAN = 'V8 Mean'
 FROG_MEAN = 'frog Mean'
 COMMAND_LINE = 'commandline'
 V8 = 'v8'
 FROG = 'frog'
 V8_AND_FROG = [V8, FROG]
 CORRECTNESS = 'Percent passing'
 COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black']
 GRAPH_OUT_DIR = 'graphs'
+
+BROWSER_PERF = 'browser-perf'
+TIME_SIZE = 'code-time-size'
+CL_PERF = 'cl-results'
+BROWSER_CORRECTNESS = 'browser-correctness'
+
 SLEEP_TIME = 200
 VERBOSE = False
 HAS_SHELL = False
 if platform.system() == 'Windows':
   # On Windows, shell must be true to get the correct environment variables.
   HAS_SHELL = True
 
 """First, some utility methods."""
 
 def run_cmd(cmd_list, outfile=None, append=False):
   """Run the specified command and print out any output to stdout.
+
   Args:
-    cmd_list a list of strings that make up the command to run
-    outfile a string indicating the name of the file that we should write stdout
-       to 
-    append True if we want to append to the file instead of overwriting it"""
+    cmd_list: a list of strings that make up the command to run
+    outfile: a string indicating the name of the file that we should write
+       stdout to 
+    append: True if we want to append to the file instead of overwriting it"""
   if VERBOSE:
     print ' '.join(cmd_list)
   out = subprocess.PIPE
   if outfile:
     mode = 'w'
     if append:
       mode = 'a'
     out = open(outfile, mode)
   p = subprocess.Popen(cmd_list, stdout = out, stderr = subprocess.PIPE,
       shell=HAS_SHELL)
   output, not_used = p.communicate();
   if output:
     print output
   return output
 
 def time_cmd(cmd):
   """Determine the amount of (real) time it takes to execute a given command."""
   start = time.time()
   run_cmd(cmd)
   return time.time() - start
 
 def sync_and_build():
   """Make sure we have the latest version of of the repo, and build it. We
   begin and end standing in DART_INSTALL_LOCATION.
+
   Returns:
     err_code = 1 if there was a problem building."""
   os.chdir(DART_INSTALL_LOCATION)
   #Revert our newly built minfrog to prevent conflicts when we update
   run_cmd(['svn', 'revert',  os.path.join(os.getcwd(), 'frog', 'minfrog')])
 
   run_cmd(['gclient', 'sync'])
   # TODO(efortuna): building the sdk locally is a band-aid until all build
   # platform SDKs are hosted in Google storage. Pull from https://sandbox.
   # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk
@@ skipping 10 matching lines @@
     if 'BUILD FAILED' in lines:
       # Someone checked in a broken build! Just stop trying to make it work
       # and wait to try again.
       print 'Broken Build'
       return 1
   return 0
 
 def ensure_output_directory(dir_name):
   """Test that the listed directory name exists, and if not, create one for
   our output to be placed.
+
   Args:
-    dir_name the directory we will create if it does not exist."""
+    dir_name: the directory we will create if it does not exist."""
   dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
       'perf_testing', dir_name)
   if not os.path.exists(dir_path):
     os.mkdir(dir_path)
     print 'Creating output directory ', dir_path
 
 def has_new_code():
   """Tests if there are any newer versions of files on the server."""
   os.chdir(DART_INSTALL_LOCATION)    
   results = run_cmd(['svn', 'st', '-u'])
@@ skipping 84 matching lines @@
         self.values_dict[platform][V8][V8_MEAN] = []
       if FROG in v8_and_or_frog_list:
         self.revision_dict[platform][FROG][FROG_MEAN] = []
         self.values_dict[platform][FROG][FROG_MEAN] = []
  
   def get_color(self):
     color = COLORS[self.color_index]
     self.color_index = (self.color_index + 1) % len(COLORS)
     return color
 
-  def syle_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, 
+  def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, 
       legend_loc, filename, platform_list, v8_and_or_frog_list, values_list, 
       should_clear_axes=True):
     """Sets style preferences for chart boilerplate that is consistent across 
     all charts, and saves the chart as a png.
+
     Args:
-      size_x the size of the printed chart, in inches, in the horizontal 
+      size_x: the size of the printed chart, in inches, in the horizontal 
         direction
-      size_y the size of the printed chart, in inches in the vertical direction
-      legend_loc the location of the legend in on the chart. See suitable
+      size_y: the size of the printed chart, in inches in the vertical direction
+      legend_loc: the location of the legend in on the chart. See suitable
         arguments for the loc argument in matplotlib
-      filename the filename that we want to save the resulting chart as
-      platform_list a list containing the platform(s) that our data has been run
-        on. (command line, firefox, chrome, etc)
-      values_list a list containing the type of data we will be graphing
+      filename: the filename that we want to save the resulting chart as
+      platform_list: a list containing the platform(s) that our data has been 
+        run on. (command line, firefox, chrome, etc)
+      values_list: a list containing the type of data we will be graphing
         (performance, percentage passing, etc)
-      should_clear_axes True if we want to create a fresh graph, instead of
+      should_clear_axes: True if we want to create a fresh graph, instead of
         plotting additional lines on the current graph."""
     if should_clear_axes:
       plt.cla() # cla = clear current axes
     for platform in platform_list:
       for f in v8_and_or_frog_list:
         for val in values_list:
           plt.plot(self.revision_dict[platform][f][val],
               self.values_dict[platform][f][val], 
               color=self.get_color(), label='%s-%s-%s' % (platform, f, val))
 
@@ skipping 79 matching lines @@
     # TODO(efortuna): You will want to make this only use a subset of the files
     # eventually.
     files = os.listdir(self.result_folder_name)
 
     for afile in files:
       if not afile.startswith('.'):
         self.process_file(afile)
 
     self.plot_results('%s.png' % self.result_folder_name)
 
-class PerformanceTestRunner(TestRunner):
+class PerformanceTest(TestRunner):
   """Super class for all performance testing."""
   def __init__(self, result_folder_name, platform_list, platform_type):
-    super(PerformanceTestRunner, self).__init__(result_folder_name, 
+    super(PerformanceTest, self).__init__(result_folder_name, 
         platform_list, get_versions(), get_benchmarks())
     self.platform_list = platform_list
     self.platform_type = platform_type
 
   def plot_all_perf(self, png_filename):
     """Create a plot that shows the performance changes of individual benchmarks
     run by V8 and generated by frog, over svn history."""
     for benchmark in get_benchmarks():
-      self.syle_and_save_perf_plot(
+      self.style_and_save_perf_plot(
           'Performance of %s over time on the %s' % (benchmark,
           self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left', 
           benchmark + png_filename, self.platform_list, get_versions(), 
           [benchmark])
 
   def plot_avg_perf(self, png_filename):
     """Generate a plot that shows the performance changes of the geomentric mean
     of V8 and frog benchmark performance over svn history."""
     (title, y_axis, size_x, size_y, loc, filename) = \
         ('Geometric Mean of benchmark %s performance' % self.platform_type, 
         'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename)
     clear_axis = True
     for platform in self.platform_list:
-      self.syle_and_save_perf_plot(title, y_axis, size_x, size_y, loc, filename, 
-          [platform], [V8], [V8_MEAN], clear_axis)
+      self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, 
+          filename, [platform], [V8], [V8_MEAN], clear_axis)
       clear_axis = False
-      self.syle_and_save_perf_plot(title, y_axis, size_x, size_y, loc, filename, 
-          [platform], [FROG], [FROG_MEAN], clear_axis)
+      self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, 
+          filename, [platform], [FROG], [FROG_MEAN], clear_axis)
 
   def plot_results(self, png_filename):
     self.plot_all_perf(png_filename)
     self.plot_avg_perf('2' + png_filename)
 
   
-class CommandLinePerformanceTestRunner(PerformanceTestRunner):
+class CommandLinePerformanceTest(PerformanceTest):
   """Run performance tests from the command line."""
 
   def __init__(self, result_folder_name):
-    super(CommandLinePerformanceTestRunner, self).__init__(result_folder_name, 
+    super(CommandLinePerformanceTest, self).__init__(result_folder_name, 
         [COMMAND_LINE], 'command line')
 
   def process_file(self, afile):
     """Pull all the relevant information out of a given tracefile.
 
     Args:
       afile is the filename string we will be processing."""
     f = open(os.path.join(self.result_folder_name, afile))
     tabulate_data = False
     revision_num = 0
@@ skipping 26 matching lines @@
   def run_tests(self):
     """Run a performance test on our updated system."""
     os.chdir('frog')
     self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing',
         self.result_folder_name, 'result' + self.cur_time)
     run_cmd(['python', os.path.join('benchmarks', 'perf_tests.py')],
         self.trace_file)
     os.chdir('..')
 
 
-class BrowserPerformanceTestRunner(PerformanceTestRunner):
+class BrowserPerformanceTest(PerformanceTest):
   """Runs performance tests, in the browser."""
 
   def __init__(self, result_folder_name):
-    super(BrowserPerformanceTestRunner, self).__init__(
+    super(BrowserPerformanceTest, self).__init__(
         result_folder_name, get_browsers(), 'browser')
 
   def run_tests(self):
     """Run a performance test in the browser."""
 
     os.chdir('frog')
     run_cmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')])
     os.chdir('..')
 
     for browser in get_browsers():
@@ skipping 50 matching lines @@
 
     f.close()
     self.calculate_geometric_mean(browser, version, revision_num)
 
   def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, 
       cleanFile=False):
       #TODO(efortuna)
       pass
 
       
-class BrowserCorrectnessTestRunner(TestRunner):
+class BrowserCorrectnessTest(TestRunner):
   def __init__(self, test_type, result_folder_name):
-    super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name,
+    super(BrowserCorrectnessTest, self).__init__(result_folder_name,
         get_browsers(), [FROG], [CORRECTNESS])
     self.test_type = test_type
 
   def run_tests(self):
-    """run a test of the latest svn revision."""
+    """Run a test of the latest svn revision."""
     system = get_os_directory()
     suffix = ''
     if platform.system() == 'Windows':
       suffix = '.exe'
     for browser in get_browsers():
       current_file = 'correctness%s-%s' % (self.cur_time, browser)
       self.trace_file = os.path.join('tools', 'testing',
           'perf_testing', self.result_folder_name, current_file)
       self.add_svn_revision_to_trace(self.trace_file)
       dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(),
           'release', 'ia32'), 'dart-sdk')
       run_cmd([os.path.join('.', 'tools', 'testing', 'bin', system, 
           'dart' + suffix), os.path.join('tools', 'test.dart'), 
           '--component=webdriver', 
           '--browser=%s' % browser, '--frog=%s' % os.path.join(dart_sdk, 'bin',
           'frogc'), '--froglib=%s' % os.path.join(dart_sdk, 'lib'), '--report',
           '--timeout=20', '--progress=color', '--mode=release', '-j1',
           self.test_type], self.trace_file, append=True)
 
   def process_file(self, afile):
     """Given a trace file, extract all the relevant information out of it to
     determine the number of correctly passing tests.
     
     Arguments:
-      afile the filename string"""
+      afile: the filename string"""
     browser = afile.rpartition('-')[2]
     f = open(os.path.join(self.result_folder_name, afile))
     revision_num = 0
     lines = f.readlines()
     total_tests = 0
     num_failed = 0
     expect_fail = 0
     for line in lines:
       if 'Total:' in line:
-        total_tests = int(line.split()[1])
+        total_tests = int(line.split('Total: ')[1].split()[0])
       if 'will be skipped' in line:
         total_tests -= int(line.split()[1])
       if 'we should fix' in line:
         expect_fail += int(line.split()[1])
       if 'Revision' in line:
         revision_num = int(line.split()[1])
       if '--- TIMEOUT ---' in line or 'FAIL:' in line or 'PASS' in line:
         # (A printed out 'PASS' indicates we incorrectly passed a negative
         # test.)
         num_failed += 1
     
     self.revision_dict[browser][FROG][CORRECTNESS] += [revision_num]
     self.values_dict[browser][FROG][CORRECTNESS] += [100.0 * 
         (((float)(total_tests - (expect_fail + num_failed))) /total_tests)]
     f.close()
 
   def plot_results(self, png_filename):
     first_time = True
     for browser in get_browsers():
-      self.syle_and_save_perf_plot('Percentage of language tests passing in '
+      self.style_and_save_perf_plot('Percentage of language tests passing in '
           'different browsers', '% of tests passed', 8, 8, 'lower left', 
           png_filename, [browser], [FROG], [CORRECTNESS], first_time) 
       first_time = False
 
 
-class CompileTimeAndSizeTestRunner(TestRunner):
+class CompileTimeAndSizeTest(TestRunner):
   """Run tests to determine how long minfrog takes to compile, and the compiled 
   file output size of some benchmarking files."""
   def __init__(self, result_folder_name):
-    super(CompileTimeAndSizeTestRunner, self).__init__(result_folder_name,
+    super(CompileTimeAndSizeTest, self).__init__(result_folder_name,
         [COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping',
         'minfrog', 'swarm', 'total'])
     self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, 
         'minfrog' : 100, 'swarm' : 100, 'total' : 100}
 
   def run_tests(self):
     os.chdir('frog')
     self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', 
         self.result_folder_name, self.result_folder_name + self.cur_time)
     
     self.add_svn_revision_to_trace(self.trace_file)    
 
-    elapsed = time_cmd([os.path.join('.', 'frog.py'), '--',
+    suffix = ''
+    if platform.system() == 'Windows':
+      suffix = '.exe'
+    elapsed = time_cmd([os.path.join('..', utils.GetBuildRoot(utils.GuessOS(), 
+    'release', 'ia32'), 'dart' + suffix), os.path.join('.', 'minfrogc.dart'),
         '--out=minfrog', 'minfrog.dart'])
     run_cmd(['echo', '%f Compiling on Dart VM in production mode in seconds' 
         % elapsed], self.trace_file, append=True)
     elapsed = time_cmd([os.path.join('.', 'minfrog'), '--out=minfrog',
         'minfrog.dart', os.path.join('tests', 'hello.dart')])
     if elapsed < self.failure_threshold['Bootstrapping']:
       #minfrog didn't compile correctly. Stop testing now, because subsequent
       #numbers will be meaningless.
       return
     size = os.path.getsize('minfrog')
     run_cmd(['echo', '%f Bootstrapping time in seconds in production mode' % 
         elapsed], self.trace_file, append=True)
     run_cmd(['echo', '%d Generated checked minfrog size' % size],
         self.trace_file, append=True)
 
-    run_cmd([os.path.join('.', 'minfrog'), ' --out=swarm-result ',
+    run_cmd([os.path.join('.', 'minfrog'), '--out=swarm-result',
         '--compile-only', os.path.join('..', 'client', 'samples', 'swarm',
         'swarm.dart')])
     swarm_size = 0
     try:
       swarm_size = os.path.getsize('swarm-result')
     except OSError:
       pass #If compilation failed, continue on running other tests.
 
     run_cmd([os.path.join('.', 'minfrog'), '--out=total-result',
-        '--compile-only', os.path.join('..', 'client', 'samples', 'total',
-        'src', 'Total.dart')])
+        '--compile-only', os.path.join('..', 'samples', 'total',
+        'client', 'Total.dart')])
     total_size = 0
     try:
       total_size = os.path.getsize('total-result')
     except OSError:
       pass #If compilation failed, continue on running other tests.
 
     run_cmd(['echo', '%d Generated checked swarm size' % swarm_size], 
         self.trace_file, append=True) 
     
     run_cmd(['echo', '%d Generated checked total size' % total_size], 
         self.trace_file, append=True) 
     os.chdir('..')
 
   def process_file(self, afile):
     """Pull all the relevant information out of a given tracefile.
+
     Args:
-      afile is the filename string we will be processing."""
+      afile: is the filename string we will be processing."""
     f = open(os.path.join(self.result_folder_name, afile))
     tabulate_data = False
     revision_num = 0
     for line in f.readlines():
       tokens = line.split()
       if 'Revision' in line:
         revision_num = int(line.split()[1])
       else:
         for metric in self.values_list:
           if metric in line:
             num = tokens[0]
             if num.find('.') == -1:
               num = int(num)
             else:
               num = float(num)
             self.values_dict[COMMAND_LINE][FROG][metric] += [num]
             self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
    
     if revision_num != 0: 
       for metric in self.values_list:
         self.revision_dict[COMMAND_LINE][FROG][metric].pop()
         self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
-        # Fill 0 if compilation failed.
+        # Fill in 0 if compilation failed.
         if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \
             self.failure_threshold[metric]:
           self.values_dict[COMMAND_LINE][FROG][metric] += [0]
           self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
 
     f.close()
 
   def plot_results(self, png_filename):
-    self.syle_and_save_perf_plot('Compiled minfrog Sizes', 
+    self.style_and_save_perf_plot('Compiled minfrog Sizes', 
         'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE],
         [FROG], ['swarm', 'total', 'minfrog'])
     self.write_html('bar', self.revision_dict[COMMAND_LINE][FROG]['minfrog'], 
         'minfrog size', self.values_dict[COMMAND_LINE][FROG]['minfrog'], '', [])
 
-    self.syle_and_save_perf_plot('Time to compile and bootstrap', 
+    self.style_and_save_perf_plot('Time to compile and bootstrap', 
         'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG],
         ['Bootstrapping', 'Compiling on Dart VM'])
     self.write_html('baz',
         self.revision_dict[COMMAND_LINE][FROG]['Bootstrapping'], 
         'Bootstrapping', self.values_dict[COMMAND_LINE][FROG]['Bootstrapping'],
         'Compiling on Dart VM', 
         self.values_dict[COMMAND_LINE][FROG]['Compiling on Dart VM'])
 
 
 def parse_args():
@@ skipping 20 matching lines @@
   if not (args.cl or args.size or args.language or args.perf):
     args.cl = args.size = args.language = args.perf = True
   return (args.cl, args.size, args.language, args.perf, args.continuous,
       args.verbose)
 
 def run_test_sequence(cl, size, language, perf):
   # The buildbot already builds and syncs to a specific revision. Don't fight
   # with it or replicate work.
   if sync_and_build() == 1:
     return # The build is broken.
+  if size:
+    CompileTimeAndSizeTest(TIME_SIZE).run()
   if cl:
-    CommandLinePerformanceTestRunner('cl-results').run()
-  if size:
-    CompileTimeAndSizeTestRunner('code-time-size').run()
+    CommandLinePerformanceTest(CL_PERF).run()
   if language:
-    BrowserCorrectnessTestRunner('language', 'browser-correctness').run()
+    BrowserCorrectnessTest('language', BROWSER_CORRECTNESS).run()
   if perf:
-    BrowserPerformanceTestRunner('browser-perf').run()
+    BrowserPerformanceTest(BROWSER_PERF).run()
 
   # TODO(efortuna): Temporarily disabled until you make a safe way to provide
   # your username/password for the uploading process.
   #upload_to_app_engine()
 
 def main():
   global VERBOSE
   (cl, size, language, perf, continuous, verbose) = parse_args()
   VERBOSE = verbose
   if continuous:
     while True:
       if has_new_code():
         run_test_sequence(cl, size, language, perf)
       else:
         time.sleep(SLEEP_TIME)
   else:
     run_test_sequence(cl, size, language, perf)
 
 if __name__ == '__main__':
   main()