Complete perf data posting.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import optparse
 import os
 from os.path import dirname, abspath
@@ skipping 159 matching lines @@
     parser.add_option('--suites', '-s', dest='suites', help='Run the specified '
                       'comma-separated test suites from set: %s' % \
                       ','.join(TestBuilder.available_suite_names()), 
                       action='store', default=None)
     parser.add_option('--forever', '-f', dest='continuous', help='Run this scri'
                       'pt forever, always checking for the next svn checkin',
                       action='store_true', default=False)
     parser.add_option('--nobuild', '-n', dest='no_build', action='store_true',
                       help='Do not sync with the repository and do not '
                       'rebuild.', default=False)
-    parser.add_option('--upload', '-u', dest='upload', action='store_true',
-                      help='Post the results of the run.', default=False)
+    parser.add_option('--noupload', '-u', dest='no_upload', action='store_true',
+                      help='Do not post the results of the run.', default=False)
+    parser.add_option('--notest', '-t', dest='no_test', action='store_true',
+                      help='Do not run the tests.', default=False)
     parser.add_option('--verbose', '-v', dest='verbose', help='Print extra '
                       'debug output', action='store_true', default=False)
 
     args, ignored = parser.parse_args()
 
     if not args.suites:
       suites = TestBuilder.available_suite_names()
     else:
       suites = []
       suitelist = args.suites.split(',')
       for name in suitelist:
         if name in TestBuilder.available_suite_names():
           suites.append(name)
         else:
           print ('Error: Invalid suite %s not in ' % name) + \
               '%s' % ','.join(TestBuilder.available_suite_names())
           sys.exit(1)
     self.suite_names = suites
     self.no_build = args.no_build
-    self.upload = args.upload
+    self.no_upload = args.no_upload
+    self.no_test = args.no_test
     self.verbose = args.verbose
     return args.continuous
 
   def run_test_sequence(self):
     """Run the set of commands to (possibly) build, run, and graph the results
     of our tests.
 
     Args:
       suite_names: The "display name" the user enters to specify which
           benchmark(s) to run.
@@ skipping 54 matching lines @@
 
   def run(self):
     """Run the benchmarks/tests from the command line and plot the
     results.
     """
     for visitor in [self.tester, self.file_processor]:
       visitor.prepare()
     
     os.chdir(DART_INSTALL_LOCATION)
     self.test_runner.ensure_output_directory(self.result_folder_name)
-    self.tester.run_tests()
+    if not self.test_runner.no_test:
+      self.tester.run_tests()
 
     os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
 
-    # TODO(efortuna): Remove trace files once uploaded. This will happen in a
-    # future CL.
     files = os.listdir(self.result_folder_name)
     for afile in files:
       if not afile.startswith('.'):
-        self.file_processor.process_file(afile)
+        if self.file_processor.process_file(afile):
+          os.remove(os.path.join(self.result_folder_name, afile))
 
 
 class Tester(object):
   """The base level visitor class that runs tests. It contains convenience 
   methods that many Tester objects use. Any class that would like to be a
   TesterVisitor must implement the run_tests() method."""
 
   def __init__(self, test):
     self.test = test
 
@@ skipping 162 matching lines @@
               os.getcwd(), 'internal', 'browserBenchmarks',
               'benchmark_page_%s.html' % version)
           self.test.test_runner.run_cmd(
               ['python', os.path.join('tools', 'testing', 'run_selenium.py'),
               '--out', file_path, '--browser', browser,
               '--timeout', '600', '--mode', 'perf'], self.test.trace_file, 
               append=True)
 
   class CommonBrowserFileProcessor(Processor):
     def process_file(self, afile):
-      """Comb through the html to find the performance results."""
+      """Comb through the html to find the performance results.
+      Returns: True if we successfullly posted our data to storage."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
                             'testing', 'perf_testing'))
+      if self.test.test_runner.no_upload:
+        return
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
       f = open(os.path.join(self.test.result_folder_name, afile))
       lines = f.readlines()
       line = ''
       i = 0
       revision_num = 0
       while '<div id="results">' not in line and i < len(lines):
         if 'Revision' in line:
           revision_num = int(line.split()[1].strip('"'))
         line = lines[i]
         i += 1
 
       if i >= len(lines) or revision_num == 0:
         # Then this run did not complete. Ignore this tracefile.
         return
 
       line = lines[i]
       i += 1
       results = []
       if line.find('<br>') > -1:
         results = line.split('<br>')
       else:
         results = line.split('<br />')
+      upload_success = True
       for result in results:
         name_and_score = result.split(':')
         if len(name_and_score) < 2:
           break
         name = name_and_score[0].strip()
         score = name_and_score[1].strip()
         if version == 'js' or version == 'v8':
           version = 'js'
-        if self.test.test_runner.upload:
-          self.report_results(name, score, browser, version, revision_num,
-                              self.SCORE)
+        upload_success = upload_success and self.report_results(
+            name, score, browser, version, revision_num, self.SCORE)
 
       f.close()
+      return upload_success
 
 
 class DromaeoTester(Tester):
   DROMAEO_BENCHMARKS = {
       'attr': ('attributes', [
           'getAttribute',
           'element.property',
           'setAttribute',
           'element.property = value']),
       'modify': ('modify', [
@@ skipping 115 matching lines @@
     def get_dromaeo_url_query(browser, version):
       if browser == 'dartium':
         version = version.replace('frog', 'dart')
       version = version.replace('_','&')
       tags = DromaeoTester.get_valid_dromaeo_tags()
       return '|'.join([ '%s&%s' % (version, tag) for tag in tags])
 
 
   class DromaeoFileProcessor(Processor):
     def process_file(self, afile):
-      """Comb through the html to find the performance results."""
+      """Comb through the html to find the performance results.
+      Returns: True if we successfullly posted our data to storage."""
+      if self.test.test_runner.no_upload:
+        return
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
 
       f = open(os.path.join(self.test.result_folder_name, afile))
       lines = f.readlines()
       i = 0
       revision_num = 0
       revision_pattern = r'Revision: (\d+)'
       suite_pattern = r'<div class="result-item done">(.+?)</ol></div>'
       result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)'
 
+      upload_success = True
       for line in lines:
         rev = re.match(revision_pattern, line.strip())
         if rev:
           revision_num = int(rev.group(1))
           continue
 
         suite_results = re.findall(suite_pattern, line)
         if suite_results:
           for suite_result in suite_results:
             results = re.findall(r'<li>(.*?)</li>', suite_result)
             if results:
               for result in results:
                 r = re.match(result_pattern, result)
                 name = DromaeoTester.legalize_filename(r.group(1).strip(':'))
                 score = float(r.group(2))
-                if self.test.test_runner.upload:
-                  self.report_results(name, score, browser, version, 
-                                      revision_num, self.SCORE)
+                upload_success = upload_success and self.report_results(
+                    name, score, browser, version, revision_num, self.SCORE)
 
       f.close()
+      return upload_success
 
 
 class DromaeoSizeTest(Test):
   """Run tests to determine the compiled file output size of Dromaeo."""
   def __init__(self, test_runner):
     super(DromaeoSizeTest, self).__init__(
         self.name(),
         ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
         DromaeoTester.DROMAEO_BENCHMARKS.keys(), test_runner, 
         self.DromaeoSizeTester(self),
@@ skipping 65 matching lines @@
             ['echo', 'Size (%s, %s): %s' % (variant, 'sum',
                                             total_size[variant])],
             self.test.trace_file, append=True)
 
 
   class DromaeoSizeProcessor(Processor):
     def process_file(self, afile):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
-        afile: is the filename string we will be processing."""
+        afile: is the filename string we will be processing.
+      Returns: True if we successfullly posted our data to storage."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
+      if self.test.test_runner.no_upload:
+        return
       f = open(os.path.join(self.test.result_folder_name, afile))
       tabulate_data = False
       revision_num = 0
       revision_pattern = r'Revision: (\d+)'
       result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)'
 
+      upload_success = True
       for line in f.readlines():
         rev = re.match(revision_pattern, line.strip())
         if rev:
           revision_num = int(rev.group(1))
           continue
 
         result = re.match(result_pattern, line.strip())
         if result:
           variant = result.group(1)
           metric = result.group(2)
           num = result.group(3)
           if num.find('.') == -1:
             num = int(num)
           else:
             num = float(num)
-          if self.test.test_runner.upload:
-            self.report_results(metric, num, 'browser', variant, revision_num,
-                                self.CODE_SIZE)
+          upload_success = upload_success and self.report_results(
+              metric, num, 'browser', variant, revision_num, self.CODE_SIZE)
 
       f.close()
+      return upload_success
   
 
 class CompileTimeAndSizeTest(Test):
   """Run tests to determine how long frogc takes to compile, and the compiled
   file output size of some benchmarking files."""
   def __init__(self, test_runner):
     """Reference to the test_runner object that notifies us when to begin
     testing."""
     super(CompileTimeAndSizeTest, self).__init__(
         self.name(), ['commandline'], ['frog'], ['swarm', 'total'],
@@ skipping 55 matching lines @@
     
       os.chdir('..')
 
 
   class CompileProcessor(Processor):
 
     def process_file(self, afile):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
-        afile: is the filename string we will be processing."""
+        afile: is the filename string we will be processing.
+      Returns: True if we successfullly posted our data to storage."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
+      if self.test.test_runner.no_upload:
+        return
       f = open(os.path.join(self.test.result_folder_name, afile))
       tabulate_data = False
       revision_num = 0
+      upload_success = True
       for line in f.readlines():
         tokens = line.split()
         if 'Revision' in line:
           revision_num = int(line.split()[1])
         else:
           for metric in self.test.values_list:
             if metric in line:
               num = tokens[0]
               if num.find('.') == -1:
                 num = int(num)
               else:
                 num = float(num)
               score_type = self.CODE_SIZE
               if 'Compiling' in metric or 'Bootstrapping' in metric:
                 score_type = self.COMPILE_TIME
-              if self.test.test_runner.upload:
-                self.report_results(metric, num, 'commandline', 'frog', 
-                                   revision_num, score_type)
+              upload_success = upload_success and self.report_results(
+                  metric, num, 'commandline', 'frog', revision_num, score_type)
 
       f.close()
+      return upload_success
 
 
 class TestBuilder(object):
   """Construct the desired test object."""
   available_suites = dict((suite.name(), suite) for suite in [
       CompileTimeAndSizeTest, CommonBrowserTest, DromaeoTest, DromaeoSizeTest])
 
   @staticmethod
   def make_test(test_name, test_runner):
     return TestBuilder.available_suites[test_name](test_runner)
@@ skipping 10 matching lines @@
     while True:
       if runner.has_new_code():
         runner.run_test_sequence()
       else:
         time.sleep(200)
   else:
     runner.run_test_sequence()
 
 if __name__ == '__main__':
   main()