Daily update script for performance numbers

tools/testing/perf_testing/create_graph.py

Index: tools/testing/perf_testing/create_graph.py
===================================================================
--- tools/testing/perf_testing/create_graph.py	(revision 3622)
+++ tools/testing/perf_testing/create_graph.py	(working copy)
@@ -37,6 +37,12 @@
 CORRECTNESS = 'Percent passing'
 COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black']
 GRAPH_OUT_DIR = 'graphs'
+
+BROWSER_PERF = 'browser-perf'
+TIME_SIZE = 'code-time-size'
+CL_PERF = 'cl-results'
+BROWSER_CORRECTNESS = 'browser-correctness'
+
 SLEEP_TIME = 200
 VERBOSE = False
 HAS_SHELL = False
@@ -48,11 +54,12 @@
 
 def run_cmd(cmd_list, outfile=None, append=False):
   """Run the specified command and print out any output to stdout.
+
   Args:
-    cmd_list a list of strings that make up the command to run
-    outfile a string indicating the name of the file that we should write stdout
-       to 
-    append True if we want to append to the file instead of overwriting it"""
+    cmd_list: a list of strings that make up the command to run
+    outfile: a string indicating the name of the file that we should write
+       stdout to 
+    append: True if we want to append to the file instead of overwriting it"""
   if VERBOSE:
     print ' '.join(cmd_list)
   out = subprocess.PIPE
@@ -77,6 +84,7 @@
 def sync_and_build():
   """Make sure we have the latest version of of the repo, and build it. We
   begin and end standing in DART_INSTALL_LOCATION.
+
   Returns:
     err_code = 1 if there was a problem building."""
   os.chdir(DART_INSTALL_LOCATION)
@@ -107,8 +115,9 @@
 def ensure_output_directory(dir_name):
   """Test that the listed directory name exists, and if not, create one for
   our output to be placed.
+
   Args:
-    dir_name the directory we will create if it does not exist."""
+    dir_name: the directory we will create if it does not exist."""
   dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
       'perf_testing', dir_name)
   if not os.path.exists(dir_path):
@@ -213,23 +222,24 @@
     self.color_index = (self.color_index + 1) % len(COLORS)
     return color
 
-  def syle_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, 
+  def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, 
       legend_loc, filename, platform_list, v8_and_or_frog_list, values_list, 
       should_clear_axes=True):
     """Sets style preferences for chart boilerplate that is consistent across 
     all charts, and saves the chart as a png.
+
     Args:
-      size_x the size of the printed chart, in inches, in the horizontal 
+      size_x: the size of the printed chart, in inches, in the horizontal 
         direction
-      size_y the size of the printed chart, in inches in the vertical direction
-      legend_loc the location of the legend in on the chart. See suitable
+      size_y: the size of the printed chart, in inches in the vertical direction
+      legend_loc: the location of the legend in on the chart. See suitable
         arguments for the loc argument in matplotlib
-      filename the filename that we want to save the resulting chart as
-      platform_list a list containing the platform(s) that our data has been run
-        on. (command line, firefox, chrome, etc)
-      values_list a list containing the type of data we will be graphing
+      filename: the filename that we want to save the resulting chart as
+      platform_list: a list containing the platform(s) that our data has been 
+        run on. (command line, firefox, chrome, etc)
+      values_list: a list containing the type of data we will be graphing
         (performance, percentage passing, etc)
-      should_clear_axes True if we want to create a fresh graph, instead of
+      should_clear_axes: True if we want to create a fresh graph, instead of
         plotting additional lines on the current graph."""
     if should_clear_axes:
       plt.cla() # cla = clear current axes
@@ -329,10 +339,10 @@
 
     self.plot_results('%s.png' % self.result_folder_name)
 
-class PerformanceTestRunner(TestRunner):
+class PerformanceTest(TestRunner):
   """Super class for all performance testing."""
   def __init__(self, result_folder_name, platform_list, platform_type):
-    super(PerformanceTestRunner, self).__init__(result_folder_name, 
+    super(PerformanceTest, self).__init__(result_folder_name, 
         platform_list, get_versions(), get_benchmarks())
     self.platform_list = platform_list
     self.platform_type = platform_type
@@ -341,7 +351,7 @@
     """Create a plot that shows the performance changes of individual benchmarks
     run by V8 and generated by frog, over svn history."""
     for benchmark in get_benchmarks():
-      self.syle_and_save_perf_plot(
+      self.style_and_save_perf_plot(
           'Performance of %s over time on the %s' % (benchmark,
           self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left', 
           benchmark + png_filename, self.platform_list, get_versions(), 
@@ -355,22 +365,22 @@
         'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename)
     clear_axis = True
     for platform in self.platform_list:
-      self.syle_and_save_perf_plot(title, y_axis, size_x, size_y, loc, filename, 
-          [platform], [V8], [V8_MEAN], clear_axis)
+      self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, 
+          filename, [platform], [V8], [V8_MEAN], clear_axis)
       clear_axis = False
-      self.syle_and_save_perf_plot(title, y_axis, size_x, size_y, loc, filename, 
-          [platform], [FROG], [FROG_MEAN], clear_axis)
+      self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, 
+          filename, [platform], [FROG], [FROG_MEAN], clear_axis)
 
   def plot_results(self, png_filename):
     self.plot_all_perf(png_filename)
     self.plot_avg_perf('2' + png_filename)
 
   
-class CommandLinePerformanceTestRunner(PerformanceTestRunner):
+class CommandLinePerformanceTest(PerformanceTest):
   """Run performance tests from the command line."""
 
   def __init__(self, result_folder_name):
-    super(CommandLinePerformanceTestRunner, self).__init__(result_folder_name, 
+    super(CommandLinePerformanceTest, self).__init__(result_folder_name, 
         [COMMAND_LINE], 'command line')
 
   def process_file(self, afile):
@@ -417,11 +427,11 @@
     os.chdir('..')
 
 
-class BrowserPerformanceTestRunner(PerformanceTestRunner):
+class BrowserPerformanceTest(PerformanceTest):
   """Runs performance tests, in the browser."""
 
   def __init__(self, result_folder_name):
-    super(BrowserPerformanceTestRunner, self).__init__(
+    super(BrowserPerformanceTest, self).__init__(
         result_folder_name, get_browsers(), 'browser')
 
   def run_tests(self):
@@ -492,14 +502,14 @@
       pass
 
       
-class BrowserCorrectnessTestRunner(TestRunner):
+class BrowserCorrectnessTest(TestRunner):
   def __init__(self, test_type, result_folder_name):
-    super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name,
+    super(BrowserCorrectnessTest, self).__init__(result_folder_name,
         get_browsers(), [FROG], [CORRECTNESS])
     self.test_type = test_type
 
   def run_tests(self):
-    """run a test of the latest svn revision."""
+    """Run a test of the latest svn revision."""
     system = get_os_directory()
     suffix = ''
     if platform.system() == 'Windows':
@@ -524,7 +534,7 @@
     determine the number of correctly passing tests.
     
     Arguments:
-      afile the filename string"""
+      afile: the filename string"""
     browser = afile.rpartition('-')[2]
     f = open(os.path.join(self.result_folder_name, afile))
     revision_num = 0
@@ -534,7 +544,7 @@
     expect_fail = 0
     for line in lines:
       if 'Total:' in line:
-        total_tests = int(line.split()[1])
+        total_tests = int(line.split('Total: ')[1].split()[0])
       if 'will be skipped' in line:
         total_tests -= int(line.split()[1])
       if 'we should fix' in line:
@@ -554,17 +564,17 @@
   def plot_results(self, png_filename):
     first_time = True
     for browser in get_browsers():
-      self.syle_and_save_perf_plot('Percentage of language tests passing in '
+      self.style_and_save_perf_plot('Percentage of language tests passing in '
           'different browsers', '% of tests passed', 8, 8, 'lower left', 
           png_filename, [browser], [FROG], [CORRECTNESS], first_time) 
       first_time = False
 
 
-class CompileTimeAndSizeTestRunner(TestRunner):
+class CompileTimeAndSizeTest(TestRunner):
   """Run tests to determine how long minfrog takes to compile, and the compiled 
   file output size of some benchmarking files."""
   def __init__(self, result_folder_name):
-    super(CompileTimeAndSizeTestRunner, self).__init__(result_folder_name,
+    super(CompileTimeAndSizeTest, self).__init__(result_folder_name,
         [COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping',
         'minfrog', 'swarm', 'total'])
     self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, 
@@ -577,7 +587,11 @@
     
     self.add_svn_revision_to_trace(self.trace_file)    
 
-    elapsed = time_cmd([os.path.join('.', 'frog.py'), '--',
+    suffix = ''
+    if platform.system() == 'Windows':
+      suffix = '.exe'
+    elapsed = time_cmd([os.path.join('..', utils.GetBuildRoot(utils.GuessOS(), 
+    'release', 'ia32'), 'dart' + suffix), os.path.join('.', 'minfrogc.dart'),
         '--out=minfrog', 'minfrog.dart'])
     run_cmd(['echo', '%f Compiling on Dart VM in production mode in seconds' 
         % elapsed], self.trace_file, append=True)
@@ -593,7 +607,7 @@
     run_cmd(['echo', '%d Generated checked minfrog size' % size],
         self.trace_file, append=True)
 
-    run_cmd([os.path.join('.', 'minfrog'), ' --out=swarm-result ',
+    run_cmd([os.path.join('.', 'minfrog'), '--out=swarm-result',
         '--compile-only', os.path.join('..', 'client', 'samples', 'swarm',
         'swarm.dart')])
     swarm_size = 0
@@ -603,8 +617,8 @@
       pass #If compilation failed, continue on running other tests.
 
     run_cmd([os.path.join('.', 'minfrog'), '--out=total-result',
-        '--compile-only', os.path.join('..', 'client', 'samples', 'total',
-        'src', 'Total.dart')])
+        '--compile-only', os.path.join('..', 'samples', 'total',
+        'client', 'Total.dart')])
     total_size = 0
     try:
       total_size = os.path.getsize('total-result')
@@ -620,8 +634,9 @@
 
   def process_file(self, afile):
     """Pull all the relevant information out of a given tracefile.
+
     Args:
-      afile is the filename string we will be processing."""
+      afile: is the filename string we will be processing."""
     f = open(os.path.join(self.result_folder_name, afile))
     tabulate_data = False
     revision_num = 0
@@ -644,7 +659,7 @@
       for metric in self.values_list:
         self.revision_dict[COMMAND_LINE][FROG][metric].pop()
         self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
-        # Fill 0 if compilation failed.
+        # Fill in 0 if compilation failed.
         if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \
             self.failure_threshold[metric]:
           self.values_dict[COMMAND_LINE][FROG][metric] += [0]
@@ -653,13 +668,13 @@
     f.close()
 
   def plot_results(self, png_filename):
-    self.syle_and_save_perf_plot('Compiled minfrog Sizes', 
+    self.style_and_save_perf_plot('Compiled minfrog Sizes', 
         'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE],
         [FROG], ['swarm', 'total', 'minfrog'])
     self.write_html('bar', self.revision_dict[COMMAND_LINE][FROG]['minfrog'], 
         'minfrog size', self.values_dict[COMMAND_LINE][FROG]['minfrog'], '', [])
 
-    self.syle_and_save_perf_plot('Time to compile and bootstrap', 
+    self.style_and_save_perf_plot('Time to compile and bootstrap', 
         'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG],
         ['Bootstrapping', 'Compiling on Dart VM'])
     self.write_html('baz',
@@ -700,14 +715,14 @@
   # with it or replicate work.
   if sync_and_build() == 1:
     return # The build is broken.
+  if size:
+    CompileTimeAndSizeTest(TIME_SIZE).run()
   if cl:
-    CommandLinePerformanceTestRunner('cl-results').run()
-  if size:
-    CompileTimeAndSizeTestRunner('code-time-size').run()
+    CommandLinePerformanceTest(CL_PERF).run()
   if language:
-    BrowserCorrectnessTestRunner('language', 'browser-correctness').run()
+    BrowserCorrectnessTest('language', BROWSER_CORRECTNESS).run()
   if perf:
-    BrowserPerformanceTestRunner('browser-perf').run()
+    BrowserPerformanceTest(BROWSER_PERF).run()
 
   # TODO(efortuna): Temporarily disabled until you make a safe way to provide
   # your username/password for the uploading process.