Get rid of selfhosted test in perf tracking and fix for uploading data.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import optparse
 import os
 from os.path import dirname, abspath
@@ skipping 93 matching lines @@
 
     # On Windows, the output directory is marked as "Read Only," which causes an
     # error to be thrown when we use shutil.rmtree. This helper function changes
     # the permissions so we can still delete the directory.
     def on_rm_error(func, path, exc_info):
       if os.path.exists(path):
         os.chmod(path, stat.S_IWRITE)
         os.unlink(path)
     # TODO(efortuna): building the sdk locally is a band-aid until all build XXX
     # platform SDKs are hosted in Google storage. Pull from https://sandbox.
-    # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk
+    # google.com/storage/?arg=dart-dump-render-tree/sdk/#dart-dump-render-tree%2Fsdk
     # eventually.
     # TODO(efortuna): Currently always building ia32 architecture because we 
     # don't have test statistics for what's passing on x64. Eliminate arch 
     # specification when we have tests running on x64, too.
     shutil.rmtree(os.path.join(os.getcwd(),
                   utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32')),
                   onerror=on_rm_error)
 
     for target in TestRunner.get_build_targets(suites):
       lines = self.run_cmd([os.path.join('.', 'tools', 'build.py'), '-m', 
@@ skipping 45 matching lines @@
     parser.add_option('--suites', '-s', dest='suites', help='Run the specified '
                       'comma-separated test suites from set: %s' % \
                       ','.join(TestBuilder.available_suite_names()), 
                       action='store', default=None)
     parser.add_option('--forever', '-f', dest='continuous', help='Run this scri'
                       'pt forever, always checking for the next svn checkin',
                       action='store_true', default=False)
     parser.add_option('--nobuild', '-n', dest='no_build', action='store_true',
                       help='Do not sync with the repository and do not '
                       'rebuild.', default=False)
+    parser.add_option('--upload', '-u', dest='upload', action='store_true',
+                      help='Post the results of the run.', default=False)
     parser.add_option('--verbose', '-v', dest='verbose', help='Print extra '
                       'debug output', action='store_true', default=False)
 
     args, ignored = parser.parse_args()
 
     if not args.suites:
       suites = TestBuilder.available_suite_names()
     else:
       suites = []
       suitelist = args.suites.split(',')
       for name in suitelist:
         if name in TestBuilder.available_suite_names():
           suites.append(name)
         else:
           print ('Error: Invalid suite %s not in ' % name) + \
               '%s' % ','.join(TestBuilder.available_suite_names())
           sys.exit(1)
     self.suite_names = suites
     self.no_build = args.no_build
+    self.upload = args.upload
     self.verbose = args.verbose
     return args.continuous
 
   def run_test_sequence(self):
     """Run the set of commands to (possibly) build, run, and graph the results
     of our tests.
 
     Args:
       suite_names: The "display name" the user enters to specify which
           benchmark(s) to run.
@@ skipping 112 matching lines @@
     elif not search_for_revision(['svn', 'info']):
       if not search_for_revision(['git', 'svn', 'info']):
         self.test.test_runner.run_cmd(['echo', 'Revision: unknown'], outfile)
 
 
 class Processor(object):
   """The base level vistor class that processes tests. It contains convenience 
   methods that many File Processor objects use. Any class that would like to be
   a ProcessorVisitor must implement the process_file() method."""
 
+  SCORE = 'Score'
+  COMPILE_TIME = 'CompileTime'
+  CODE_SIZE = 'CodeSize'
+
   def __init__(self, test):
     self.test = test
 
   def prepare(self):
     """Perform any initial setup required before the test is run."""
     pass
 
   def report_results(self, benchmark_name, score, platform, variant, 
-                     revision_number):
+                     revision_number, metric):
     """Store the results of the benchmark run.
     Args:
       benchmark_name: The name of the individual benchmark.
       score: The numerical value of this benchmark.
       platform: The platform the test was run on (firefox, command line, etc).
       variant: Specifies whether the data was about generated Frog, js, a
           combination of both, or Dart depending on the test.
       revision_number: The revision of the code (and sometimes the revision of
           dartium).
   
     Returns: True if the post was successful."""
+    # TODO(efortuna): delete results file if returns True.
     return post_results.report_results(benchmark_name, score, platform, variant,
-                                revision_number)
+                                revision_number, metric)
 
 
 class RuntimePerformanceTest(Test):
   """Super class for all runtime performance testing."""
 
   def __init__(self, result_folder_name, platform_list, platform_type,
                versions, benchmarks, test_runner, tester, file_processor,
                build_targets=['create_sdk']):
     """Args:
         result_folder_name: The name of the folder where a tracefile of
@@ skipping 123 matching lines @@
       else:
         results = line.split('<br />')
       for result in results:
         name_and_score = result.split(':')
         if len(name_and_score) < 2:
           break
         name = name_and_score[0].strip()
         score = name_and_score[1].strip()
         if version == 'js' or version == 'v8':
           version = 'js'
-        self.report_results(name, score, browser, version, revision_num)
+        if self.test.test_runner.upload:
+          self.report_results(name, score, browser, version, revision_num,
+                              self.SCORE)
 
       f.close()
 
 
 class DromaeoTester(Tester):
   DROMAEO_BENCHMARKS = {
       'attr': ('attributes', [
           'getAttribute',
           'element.property',
           'setAttribute',
@@ skipping 145 matching lines @@
 
         suite_results = re.findall(suite_pattern, line)
         if suite_results:
           for suite_result in suite_results:
             results = re.findall(r'<li>(.*?)</li>', suite_result)
             if results:
               for result in results:
                 r = re.match(result_pattern, result)
                 name = DromaeoTester.legalize_filename(r.group(1).strip(':'))
                 score = float(r.group(2))
-                self.report_results(name, score, browser, version, revision_num)
+                if self.test.test_runner.upload:
+                  self.report_results(name, score, browser, version, 
+                                      revision_num, self.SCORE)
 
       f.close()
 
 
 class DromaeoSizeTest(Test):
   """Run tests to determine the compiled file output size of Dromaeo."""
   def __init__(self, test_runner):
     super(DromaeoSizeTest, self).__init__(
         self.name(),
         ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'],
@@ skipping 91 matching lines @@
 
         result = re.match(result_pattern, line.strip())
         if result:
           variant = result.group(1)
           metric = result.group(2)
           num = result.group(3)
           if num.find('.') == -1:
             num = int(num)
           else:
             num = float(num)
-          self.report_results(metric, num, 'browser', variant, revision_num)
+          if self.test.test_runner.upload:
+            self.report_results(metric, num, 'browser', variant, revision_num,
+                                self.CODE_SIZE)
 
       f.close()
   
 
 class CompileTimeAndSizeTest(Test):
-  """Run tests to determine how long minfrog takes to compile, and the compiled
+  """Run tests to determine how long frogc takes to compile, and the compiled
   file output size of some benchmarking files."""
   def __init__(self, test_runner):
     """Reference to the test_runner object that notifies us when to begin
     testing."""
     super(CompileTimeAndSizeTest, self).__init__(
-        self.name(), ['commandline'], ['frog'], 
-        ['Compiling on Dart VM', 'Bootstrapping', 'minfrog', 'swarm', 'total'],
+        self.name(), ['commandline'], ['frog'], ['swarm', 'total'],
         test_runner, self.CompileTester(self),
         self.CompileProcessor(self))
     self.dart_compiler = os.path.join(
         DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(),
         'release', 'ia32'), 'dart-sdk', 'bin', 'frogc')
     _suffix = ''
     if platform.system() == 'Windows':
       _suffix = '.exe'
     self.dart_vm = os.path.join(
         DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), 
         'release', 'ia32'), 'dart-sdk', 'bin','dart' + _suffix)
-    self.failure_threshold = {
-        'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, 'minfrog' : 100, 
-        'swarm' : 100, 'total' : 100}
+    self.failure_threshold = {'swarm' : 100, 'total' : 100}
 
   @staticmethod
   def name():
     return 'time-size'
 
   class CompileTester(Tester):
     def run_tests(self):
       os.chdir('frog')
       self.test.trace_file = os.path.join(
           '..', 'tools', 'testing', 'perf_testing', 
           self.test.result_folder_name,
           self.test.result_folder_name + self.test.cur_time)
 
       self.add_svn_revision_to_trace(self.test.trace_file)
 
-      elapsed = self.test.test_runner.time_cmd(
-          [self.test.dart_vm, os.path.join('.', 'minfrogc.dart'),
-          '--out=minfrog', 'minfrog.dart'])
       self.test.test_runner.run_cmd(
-          ['echo', '%f Compiling on Dart VM in production mode in seconds'
-          % elapsed], self.test.trace_file, append=True)
-      elapsed = self.test.test_runner.time_cmd(
-          [os.path.join('.', 'minfrog'), '--out=minfrog', 'minfrog.dart', 
-          os.path.join('tests', 'hello.dart')])
-      if elapsed < self.test.failure_threshold['Bootstrapping']:
-        #minfrog didn't compile correctly. Stop testing now, because subsequent
-        #numbers will be meaningless.
-        return
-      size = os.path.getsize('minfrog')
-      self.test.test_runner.run_cmd(
-          ['echo', '%f Bootstrapping time in seconds in production mode' %
-          elapsed], self.test.trace_file, append=True)
-      self.test.test_runner.run_cmd(
-          ['echo', '%d Generated checked minfrog size' % size],
-          self.test.trace_file, append=True)
-
-      self.test.test_runner.run_cmd(
-          [self.test.dart_compiler, '--out=swarm-result',
+          [self.test.dart_vm, 'frogc.dart', '--out=swarm-result',
           os.path.join('..', 'samples', 'swarm',
           'swarm.dart')])
 
       swarm_size = 0
       try:
         swarm_size = os.path.getsize('swarm-result')
       except OSError:
         pass #If compilation failed, continue on running other tests.
 
       self.test.test_runner.run_cmd(
-          [self.test.dart_compiler, '--out=total-result',
+          [self.test.dart_vm, 'frogc.dart', '--out=total-result',
           os.path.join('..', 'samples', 'total',
           'client', 'Total.dart')])
       total_size = 0
       try:
         total_size = os.path.getsize('total-result')
       except OSError:
         pass #If compilation failed, continue on running other tests.
 
       self.test.test_runner.run_cmd(
           ['echo', '%d Generated checked swarm size' % swarm_size],
           self.test.trace_file, append=True)
 
       self.test.test_runner.run_cmd(
           ['echo', '%d Generated checked total size' % total_size],
           self.test.trace_file, append=True)
     
-      #Revert our newly built minfrog to prevent conflicts when we update
-      self.test.test_runner.run_cmd(
-          ['svn', 'revert',  os.path.join(os.getcwd(), 'minfrog')])
       os.chdir('..')
 
 
   class CompileProcessor(Processor):
 
     def process_file(self, afile):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
         afile: is the filename string we will be processing."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
       f = open(os.path.join(self.test.result_folder_name, afile))
       tabulate_data = False
       revision_num = 0
       for line in f.readlines():
         tokens = line.split()
         if 'Revision' in line:
           revision_num = int(line.split()[1])
         else:
           for metric in self.test.values_list:
             if metric in line:
               num = tokens[0]
               if num.find('.') == -1:
                 num = int(num)
               else:
                 num = float(num)
-              self.report_results(metric, num, 'commandline', 'frog', 
-                                 revision_num)
+              score_type = self.CODE_SIZE
+              if 'Compiling' in metric or 'Bootstrapping' in metric:
+                score_type = self.COMPILE_TIME
+              if self.test.test_runner.upload:
+                self.report_results(metric, num, 'commandline', 'frog', 
+                                   revision_num, score_type)

[ricow1, 2012/04/25 08:42:18]

indentation off

 
       f.close()
 
 
 class TestBuilder(object):
   """Construct the desired test object."""
   available_suites = dict((suite.name(), suite) for suite in [
       CompileTimeAndSizeTest, CommonBrowserTest, DromaeoTest, DromaeoSizeTest])
 
   @staticmethod
@@ skipping 12 matching lines @@
     while True:
       if runner.has_new_code():
         runner.run_test_sequence()
       else:
         time.sleep(200)
   else:
     runner.run_test_sequence()
 
 if __name__ == '__main__':
   main()