More tweaks to perf script

tools/testing/perf_testing/run_perf_tests.py

Index: tools/testing/perf_testing/run_perf_tests.py
===================================================================
--- tools/testing/perf_testing/run_perf_tests.py	(revision 10662)
+++ tools/testing/perf_testing/run_perf_tests.py	(working copy)
@@ -11,6 +11,7 @@
 from os.path import dirname, abspath
 import pickle
 import platform
+import random
 import re
 import shutil
 import stat
@@ -58,7 +59,7 @@
     if outfile:
       mode = 'w'
       if append:
-        mode = 'a'
+        mode = 'a+'
       out = open(outfile, mode)
       if append:
         # Annoying Windows "feature" -- append doesn't actually append unless
@@ -94,15 +95,15 @@
           else:
             os.remove(to_remove)
 
-  def get_archive(archive_name):
+  def get_archive(self, archive_name):
     """Wrapper around the pulling down a specific archive from Google Storage.
     Adds a specific revision argument as needed.
     Returns: The stderr from running this command."""
     cmd = ['python', os.path.join(DART_REPO_LOC, 'tools', 'get_archive.py'),
            archive_name]
     if self.current_revision_num != -1:
-      cmd += ['-r', revision_num]
-    _, stderr = self.test.test_runner.run_cmd(cmd)
+      cmd += ['-r', self.current_revision_num]
+    _, stderr = self.run_cmd(cmd)
     return stderr
 
   def sync_and_build(self, suites, revision_num=''):
@@ -135,8 +136,10 @@
         revision_num = search_for_revision(['git', 'svn', 'info'])
 
     self.current_revision_num = revision_num
-    stderr = get_archive('sdk')
-    if not os.path.exists(get_archive_path) or 'InvalidUriError' in stderr:
+    stderr = self.get_archive('sdk')
+    if not os.path.exists(os.path.join(
+        DART_REPO_LOC, 'tools', 'get_archive.py')) \
+        or 'InvalidUriError' in stderr:
       # Couldn't find the SDK on Google Storage. Build it locally.
 
       # On Windows, the output directory is marked as "Read Only," which causes
@@ -247,7 +250,7 @@
     success = True
     if not self.no_build and self.sync_and_build(suites, revision_num) == 1:
       return -1 # The build is broken.
-    
+
     for name in self.suite_names:
       for run in range(num_reruns):
         suites += [TestBuilder.make_test(name, self)]
@@ -424,7 +427,7 @@
                                        revision_number, metric)
 
   def calculate_geometric_mean(self, platform, variant, svn_revision):
-    """Calculate the aggregate geometric mean for JS and frog benchmark sets,
+    """Calculate the aggregate geometric mean for JS and dart2js benchmark sets,
     given two benchmark dictionaries."""
     geo_mean = 0		
     if self.test.is_valid_combination(platform, variant):
@@ -498,7 +501,7 @@
       test_runner: Reference to the object that notifies us when to run."""
     super(CommonBrowserTest, self).__init__(
         self.name(), BrowserTester.get_browsers(False),
-        'browser', ['js', 'frog', 'dart2js'],
+        'browser', ['js', 'dart2js'],
         self.get_standalone_benchmarks(), test_runner, 
         self.CommonBrowserTester(self),
         self.CommonBrowserFileProcessor(self))
@@ -665,7 +668,7 @@
 
   @staticmethod
   def get_dromaeo_versions():
-    return ['js', 'dart2js_dom', 'dart2js_html']
+    return ['js', 'dart2js_html']
 
 
 class DromaeoTest(RuntimePerformanceTest):
@@ -707,7 +710,6 @@
       """
       current_dir = os.getcwd()
       self.test.test_runner.get_archive('chromedriver')
-      self.test.test_runner.run_cmd(['python', os.path.join(
       path = os.environ['PATH'].split(os.pathsep)
       orig_chromedriver_path = os.path.join(DART_REPO_LOC, 'tools', 'testing',
                                             'orig-chromedriver')
@@ -848,248 +850,10 @@
       self.calculate_geometric_mean(browser, version, revision_num)
       return upload_success
 
-
-class DromaeoSizeTest(Test):
-  """Run tests to determine the compiled file output size of Dromaeo."""
-  def __init__(self, test_runner):
-    super(DromaeoSizeTest, self).__init__(
-        self.name(),
-        ['commandline'], ['dart', 'frog_dom', 'frog_html',
-         'frog_htmlidiomatic'],
-        DromaeoTester.DROMAEO_BENCHMARKS.keys(), test_runner, 
-        self.DromaeoSizeTester(self),
-        self.DromaeoSizeProcessor(self), extra_metrics=['sum'])
-  
-  @staticmethod
-  def name():
-    return 'dromaeo-size'
-
-
-  class DromaeoSizeTester(DromaeoTester):
-    def run_tests(self):
-      # Build tests.
-      dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
-      current_path = os.getcwd()
-      os.chdir(dromaeo_path)
-      self.test.test_runner.run_cmd(
-          ['python', os.path.join('generate_dart2js_tests.py')])
-      self.test.test_runner.get_archive('dartium')
-      os.chdir(current_path)
-
-      self.test.trace_file = os.path.join(TOP_LEVEL_DIR,
-          'tools', 'testing', 'perf_testing', self.test.result_folder_name,
-          self.test.result_folder_name + self.test.cur_time)
-      self.add_svn_revision_to_trace(self.test.trace_file)
-
-      variants = [
-          ('frog_dom', ''),
-          ('frog_html', '-html'),
-          ('frog_htmlidiomatic', '-htmlidiomatic')]
-
-      test_path = os.path.join(dromaeo_path, 'tests')
-      frog_path = os.path.join(test_path, 'frog')
-      total_size = {}
-      for (variant, _) in variants:
-        total_size[variant] = 0
-      total_dart_size = 0
-      for suite in DromaeoTester.DROMAEO_BENCHMARKS.keys():
-        dart_size = 0
-        try:
-          dart_size = os.path.getsize(os.path.join(test_path,
-                                                   'dom-%s.dart' % suite))
-        except OSError:
-          pass #If compilation failed, continue on running other tests.
-
-        total_dart_size += dart_size
-        self.test.test_runner.run_cmd(
-            ['echo', 'Size (dart, %s): %s' % (suite, str(dart_size))],
-            self.test.trace_file, append=True)
-
-        for (variant, suffix) in variants:
-          name = 'dom-%s%s.dart.js' % (suite, suffix)
-          js_size = 0
-          try:
-            # TODO(vsm): Strip comments at least.  Consider compression.
-            js_size = os.path.getsize(os.path.join(frog_path, name))
-          except OSError:
-            pass #If compilation failed, continue on running other tests.
-
-          total_size[variant] += js_size
-          self.test.test_runner.run_cmd(
-              ['echo', 'Size (%s, %s): %s' % (variant, suite, str(js_size))],
-              self.test.trace_file, append=True)
-
-      self.test.test_runner.run_cmd(
-          ['echo', 'Size (dart, %s): %s' % (total_dart_size,
-                                            self.test.extra_metrics[0])],
-          self.test.trace_file, append=True)
-      for (variant, _) in variants:
-        self.test.test_runner.run_cmd(
-            ['echo', 'Size (%s, %s): %s' % (variant, self.test.extra_metrics[0],
-                                            total_size[variant])],
-            self.test.trace_file, append=True)
-
-
-  class DromaeoSizeProcessor(Processor):
-    def process_file(self, afile, should_post_file):
-      """Pull all the relevant information out of a given tracefile.
-
-      Args:
-        afile: is the filename string we will be processing.
-      Returns: True if we successfully posted our data to storage."""
-      os.chdir(os.path.join(TOP_LEVEL_DIR, 'tools',
-          'testing', 'perf_testing'))
-      f = self.open_trace_file(afile, should_post_file)
-      tabulate_data = False
-      revision_num = 0
-      revision_pattern = r'Revision: (\d+)'
-      result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)'
-
-      upload_success = True
-      for line in f.readlines():
-        rev = re.match(revision_pattern, line.strip())
-        if rev:
-          revision_num = int(rev.group(1))
-          continue
-
-        result = re.match(result_pattern, line.strip())
-        if result:
-          variant = result.group(1)
-          metric = result.group(2)
-          num = result.group(3)
-          if num.find('.') == -1:
-            num = int(num)
-          else:
-            num = float(num)
-          self.test.values_dict['commandline'][variant][metric] += [num]
-          self.test.revision_dict['commandline'][variant][metric] += \
-              [revision_num]
-          if not self.test.test_runner.no_upload and should_post_file:
-            upload_success = upload_success and self.report_results(
-                metric, num, 'commandline', variant, revision_num,
-                self.get_score_type(metric))
-          else:
-            upload_success = False
-
-      f.close()
-      return upload_success
-    
-    def get_score_type(self, metric):
-      return self.CODE_SIZE
-
-
-class CompileTimeAndSizeTest(Test):
-  """Run tests to determine how long frogc takes to compile, and the compiled
-  file output size of some benchmarking files.
-  Note: This test is now 'deprecated' since frog is no longer in the sdk. We
-  just return the last numbers found for frog."""
-  def __init__(self, test_runner):
-    """Reference to the test_runner object that notifies us when to begin
-    testing."""
-    super(CompileTimeAndSizeTest, self).__init__(
-        self.name(), ['commandline'], ['dart2js'], ['swarm'],
-        test_runner, self.CompileTester(self),
-        self.CompileProcessor(self))
-    self.dart_compiler = os.path.join(
-        DART_REPO_LOC, utils.GetBuildRoot(utils.GuessOS(),
-        'release', 'ia32'), 'dart-sdk', 'bin', 'dart2js')
-    _suffix = ''
-    if platform.system() == 'Windows':
-      _suffix = '.exe'
-    self.failure_threshold = {'swarm' : 100}
-
-  @staticmethod
-  def name():
-    return 'time-size'
-
-  class CompileTester(Tester):
-    def run_tests(self):
-      self.test.trace_file = os.path.join(TOP_LEVEL_DIR,
-          'tools', 'testing', 'perf_testing', 
-          self.test.result_folder_name,
-          self.test.result_folder_name + self.test.cur_time)
-
-      self.add_svn_revision_to_trace(self.test.trace_file)
-
-      self.test.test_runner.run_cmd(
-          ['./xcodebuild/ReleaseIA32/dart-sdk/dart2js', '-c', '-o',
-           'swarm-result', os.path.join('samples', 'swarm', 'swarm.dart')])
-      swarm_size = 0
-      try:
-        swarm_size = os.path.getsize('swarm-result')
-      except OSError:
-        pass #If compilation failed, continue on running other tests.
-
-      self.test.test_runner.run_cmd(
-          ['echo', '%d Generated checked swarm size' % swarm_size],
-          self.test.trace_file, append=True)
-
-  class CompileProcessor(Processor):
-    def process_file(self, afile, should_post_file):
-      """Pull all the relevant information out of a given tracefile.
-
-      Args:
-        afile: is the filename string we will be processing.
-      Returns: True if we successfully posted our data to storage."""
-      os.chdir(os.path.join(TOP_LEVEL_DIR, 'tools',
-          'testing', 'perf_testing'))
-      f = self.open_trace_file(afile, should_post_file)
-      tabulate_data = False
-      revision_num = 0
-      upload_success = True
-      for line in f.readlines():
-        tokens = line.split()
-        if 'Revision' in line:
-          revision_num = int(line.split()[1])
-        else:
-          for metric in self.test.values_list:
-            if metric in line:
-              num = tokens[0]
-              if num.find('.') == -1:
-                num = int(num)
-              else:
-                num = float(num)
-              self.test.values_dict['commandline']['dart2js'][metric] += [num]
-              self.test.revision_dict['commandline']['dart2js'][metric] += \
-                  [revision_num]
-              score_type = self.get_score_type(metric)
-              if not self.test.test_runner.no_upload and should_post_file:
-                if num < self.test.failure_threshold[metric]:
-                  num = 0
-                upload_success = upload_success and self.report_results(
-                    metric, num, 'commandline', 'dart2js', revision_num,
-                    score_type)
-              else:
-                upload_success = False
-      if revision_num != 0:
-        for metric in self.test.values_list:
-          try:
-            self.test.revision_dict['commandline']['dart2js'][metric].pop()
-            self.test.revision_dict['commandline']['dart2js'][metric] += \
-                [revision_num]
-            # Fill in 0 if compilation failed.
-            if self.test.values_dict['commandline']['dart2js'][metric][-1] < \
-                self.test.failure_threshold[metric]:
-              self.test.values_dict['commandline']['dart2js'][metric] += [0]
-              self.test.revision_dict['commandline']['dart2js'][metric] += \
-                  [revision_num]
-          except IndexError:
-            # We tried to pop from an empty list. This happens if the first
-            # trace file we encounter is incomplete.
-            pass
-
-      f.close()
-      return upload_success
-
-    def get_score_type(self, metric):
-      if 'Compiling' in metric or 'Bootstrapping' in metric:
-        return self.COMPILE_TIME
-      return self.CODE_SIZE
-
 class TestBuilder(object):
   """Construct the desired test object."""
   available_suites = dict((suite.name(), suite) for suite in [
-      CompileTimeAndSizeTest, CommonBrowserTest, DromaeoTest, DromaeoSizeTest])
+      CommonBrowserTest, DromaeoTest])
 
   @staticmethod
   def make_test(test_name, test_runner):
@@ -1129,6 +893,79 @@
   f.close()
   return result_set
 
+def fill_in_back_history(results_set, runner):
+  """ Fill in back history performance data. This is done one of two ways, with
+  equal probability of trying each way (falling back on the sequential version
+  as our data becomes more densely populated)."""
+  has_run_extra = False
+  os.chdir(DART_REPO_LOC)
+  revision_num = int(search_for_revision(['svn', 'info']))
+  if revision_num == -1:
+    revision_num = int(search_for_revision(['git', 'svn', 'info']))

[vsm, 2012/08/15 15:27:32]

Nit: I'd fold the previous 4 lines into search_for_revision and call something
like:
  revision_num = search_for_revision(DART_REPO_LOC)
here. 

[Emily Fortuna, 2012/08/15 22:10:10]

Done.

+  os.chdir(TOP_LEVEL_DIR)
+
+  def try_to_run_additional(revision_number):
+    """Determine the number of results we have stored for a particular revision
+    number, and if it is less than 10, run some extra tests.
+    Args: 
+      - revision_number: the revision whose performance we want to potentially
+        test."""
+    print 'running %s~~~~~~~~~' % revision_number
+    a_test = TestBuilder.make_test(runner.suite_names[0], runner)
+    benchmark_name = a_test.values_list[0]
+    platform_name = a_test.platform_list[0]
+    variant = a_test.values_dict[platform_name].keys()[0]
+    num_results = post_results.get_num_results(benchmark_name,
+        platform_name, variant, revision_number,
+        a_test.file_processor.get_score_type(benchmark_name))
+    if 10 - num_results < 2 and num_results >= 0:

[vsm, 2012/08/15 15:27:32]

This looks fishy.  "10 - num_results < 2" is equivalent to "num_results > 8",
which makes the second part redundant.

[Emily Fortuna, 2012/08/15 22:10:10]

num_results returns -1 if it has an error connecting and getting data, in which
case reruns = 10-(-1) ==> 11 which is not what we want! This handles the case if
it can't connect, it just runs two additional perf tests for good measure
(because if it can't connect, it's likely to not connect for a little bit, and I
figured running tests was better than sitting idle). I'll rewrite >=0 to != -1
so that situation is a little more obvious.

+      reruns = 10 - num_results
+    else:
+      reruns = 2
+    run = runner.run_test_sequence(revision_num=str(revision_number),
+        num_reruns=reruns)
+    if run == 0 and num_results + reruns >= 10:
+      results_set = update_set_of_done_cls(revision_number)
+    else:
+      return False
+    return True
+
+  if random.choice([True, False]):
+    # Select a random CL number, with greater likelihood of selecting a CL in
+    # the more recent history than the distant past (using a simplified weighted
+    # bucket algorithm). If that CL has less than 10 runs, run additional. If it
+    # already has 10 runs, look for another CL number that is not yet have all
+    # of its additional runs (do this up to 15 times).
+    tries = 0
+    thousands_list = range(1, int(revision_num)/1000 + 1)
+    weighted_total = sum(thousands_list)
+    generated_random_number = random.randint(0, weighted_total - 1)
+    for i in list(reversed(thousands_list)):
+      thousands = thousands_list[i - 1]
+      weighted_total -= thousands_list[i - 1]
+      if weighted_total <= generated_random_number:
+        break
+    while tries < 15 and not has_run_extra:
+      rev = thousands * 1000 + random.randrange(0,
+          int(revision_num) -
+          int(math.pow(10, int(math.floor(math.log10(int(revision_num))))) + 1))
+      has_run_extra = try_to_run_additional(rev)
+      tries += 1
+
+  if not has_run_extra:
+    # Try to get up to 10 runs of each CL, starting with the most recent
+    # CL that does not yet have 10 runs. But only perform a set of extra
+    # runs at most 2 at a time before
+    # checking to see if new code has been checked in.
+    while revision_num > 0 and not has_run_extra:
+      if revision_num not in results_set:
+        has_run_extra = try_to_run_additional(revision_num)
+      revision_num -= 1
+  if not has_run_extra:
+    # No more extra back-runs to do (for now). Wait for new code.
+    time.sleep(200)
+  return results_set
+
 def main():
   runner = TestRunner()
   continuous = runner.parse_args()
@@ -1147,35 +984,7 @@
       if runner.has_new_code():
         runner.run_test_sequence()
       else:
-        # Try to get up to 10 runs of each CL, starting with the most recent CL
-        # that does not yet have 10 runs. But only perform a set of extra runs
-        # at most 10 at a time (get all the extra runs for one CL) before
-        # checking to see if new code has been checked in.
-        has_run_extra = False
-        revision_num = int(search_for_revision(['svn', 'info']))
-        if revision_num == -1:
-          revision_num = int(search_for_revision(['git', 'svn', 'info']))
-        
-        # No need to track the performance before revision 3000. That's way in
-        # the past.
-        while revision_num > 3000 and not has_run_extra:
-          if revision_num not in results_set:
-            a_test = TestBuilder.make_test(runner.suite_names[0], runner)
-            benchmark_name = a_test.values_list[0]
-            platform_name = a_test.platform_list[0]
-            variant = a_test.values_dict[platform_name].keys()[0]
-            number_of_results = post_results.get_num_results(benchmark_name,
-                platform_name, variant, revision_num,
-                a_test.file_processor.get_score_type(benchmark_name))
-            if number_of_results < 10 and number_of_results >= 0:
-              run = runner.run_test_sequence(revision_num=str(revision_num),
-                  num_reruns=(10-number_of_results))
-              if run == 0:
-                has_run_extra = True
-                results_set = update_set_of_done_cls(revision_num)
-          revision_num -= 1
-        # No more extra back-runs to do (for now). Wait for new code.
-        time.sleep(200)
+        results_set = fill_in_back_history(results_set, runner)
   else:
     runner.run_test_sequence()