Additional run_perf_tests.py tweaks.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
-import math             
+import math
 import optparse
 import os
 from os.path import dirname, abspath
 import pickle
 import platform
 import random
 import re
 import shutil
 import stat
 import subprocess
@@ skipping 69 matching lines @@
 
   def ClearOutUnversionedFiles(self):
     """Remove all files that are unversioned by svn."""
     if os.path.exists(DART_REPO_LOC):
       os.chdir(DART_REPO_LOC)
       results, _ = self.RunCmd(['svn', 'st'])
       for line in results.split('\n'):
         if line.startswith('?'):
           to_remove = line.split()[1]
           if os.path.isdir(to_remove):
-            shutil.rmtree(to_remove)#, ignore_errors=True)
+            shutil.rmtree(to_remove, ignore_errors=True)
           else:
             os.remove(to_remove)
         elif any(line.startswith(status) for status in ['A', 'M', 'C', 'D']):
           self.RunCmd(['svn', 'revert', line.split()[1]])
 
   def GetArchive(self, archive_name):
     """Wrapper around the pulling down a specific archive from Google Storage.
     Adds a specific revision argument as needed.
-    Returns: The stdout and stderr from running this command."""
+    Returns: A tuple of a boolean (True if we successfully downloaded the
+    binary), and the stdout and stderr from running this command."""
+    num_fails = 0
+    success = True
     while True:
       cmd = ['python', os.path.join(DART_REPO_LOC, 'tools', 'get_archive.py'),
              archive_name]
       if int(self.current_revision_num) != -1:
         cmd += ['-r', str(self.current_revision_num)]
       stdout, stderr = self.RunCmd(cmd)
-      if 'Please try again later' in stdout:
+      if 'Please try again later' in stdout and num_fails < 20:
         time.sleep(100)
+        num_fails += 1
       else:
         break
-    return (stdout, stderr)
+    return (num_fails < 20, stdout, stderr)
 
   def _Sync(self, revision_num=None):
     """Update the repository to the latest or specified revision."""
     os.chdir(dirname(DART_REPO_LOC))
     self.ClearOutUnversionedFiles()
     if not revision_num:
       self.RunCmd(['gclient', 'sync'])
     else:
       self.RunCmd(['gclient', 'sync', '-r', str(revision_num), '-t'])
     
@@ skipping 12 matching lines @@
     Args:
       suites: The set of suites that we wish to build.
 
     Returns:
       err_code = 1 if there was a problem building."""
     self._Sync(revision_num)
     if not revision_num:
       revision_num = SearchForRevision()
 
     self.current_revision_num = revision_num
-    stdout, stderr = self.GetArchive('sdk')
+    success, stdout, stderr = self.GetArchive('sdk')
     if (not os.path.exists(os.path.join(
-        DART_REPO_LOC, 'tools', 'get_archive.py'))
+        DART_REPO_LOC, 'tools', 'get_archive.py')) or not success
         or 'InvalidUriError' in stderr or "Couldn't download" in stdout):
       # Couldn't find the SDK on Google Storage. Build it locally.
 
       # On Windows, the output directory is marked as "Read Only," which causes
       # an error to be thrown when we use shutil.rmtree. This helper function
       # changes the permissions so we can still delete the directory.
       def on_rm_error(func, path, exc_info):
         if os.path.exists(path):
           os.chmod(path, stat.S_IWRITE)
           os.unlink(path)
@@ skipping 86 matching lines @@
     if revision_num:
       return (HasPerfAffectingResults(GetFileList(
           revision_num)), revision_num)
     else:
       latest_interesting_server_rev = None
       while not latest_interesting_server_rev:
         results, _ = self.RunCmd(['svn', 'st', '-u'], std_in='p\r\n')
         if len(results.split('\n')) >= 2:
           latest_interesting_server_rev = int(
               results.split('\n')[-2].split()[-1])
-          print 'success'
-        else:
-          print 'hrmmmm'
       if self.backfill:
         done_cls = list(UpdateSetOfDoneCls())
         done_cls.sort()
         if done_cls:
           last_done_cl = int(done_cls[-1])
         else:
           last_done_cl = EARLIEST_REVISION
         while latest_interesting_server_rev >= last_done_cl:
           file_list = GetFileList(latest_interesting_server_rev)
           if HasPerfAffectingResults(file_list):
@@ skipping 142 matching lines @@
         for extra_metric in extra_metrics:              
           self.revision_dict[platform][f][extra_metric] = []            
           self.values_dict[platform][f][extra_metric] = []
 
   def IsValidCombination(self, platform, variant):
     """Check whether data should be captured for this platform/variant
     combination.
     """
     # TODO(vsm): This avoids a bug in 32-bit Chrome (dartium)
     # running JS dromaeo.
-    if variant == 'js':
-      return False
     if platform == 'dartium' and variant == 'js':
       return False
     if (platform == 'safari' and variant == 'dart2js' and
         int(self.test_runner.current_revision_num) < 10193):
       # In revision 10193 we fixed a bug that allows Safari 6 to run dart2js
       # code. Since we can't change the Safari version on the machine, we're
       # just not running
       # for this case.
       return False
     return True
@@ skipping 153 matching lines @@
           file_processor)
     self.platform_list = platform_list
     self.platform_type = platform_type
     self.versions = versions
     self.benchmarks = benchmarks 
 
 
 class BrowserTester(Tester):
   @staticmethod
   def GetBrowsers(add_dartium=True):
-    browsers = ['ff']#, 'chrome']
+    browsers = ['ff', 'chrome']
     if add_dartium:
-      pass#browsers += ['dartium']
+      browsers += ['dartium']
     has_shell = False
     if platform.system() == 'Darwin':
       browsers += ['safari']
     if platform.system() == 'Windows':
-      #browsers += ['ie']
+      browsers += ['ie']
       has_shell = True
     return browsers
 
 
 class CommonBrowserTest(RuntimePerformanceTest):
   """Runs this basic performance tests (Benchpress, some V8 benchmarks) in the
   browser."""
 
   def __init__(self, test_runner):
     """Args:
@@ skipping 40 matching lines @@
               '--out', file_path, '--browser', browser,
               '--timeout', '600', '--mode', 'perf'], self.test.trace_file, 
               append=True)
 
   class CommonBrowserFileProcessor(Processor):
 
     def ProcessFile(self, afile, should_post_file):
       """Comb through the html to find the performance results.
       Returns: True if we successfully posted our data to storage and/or we can
           delete the trace file."""
-      print afile
       os.chdir(os.path.join(TOP_LEVEL_DIR, 'tools',
                             'testing', 'perf_testing'))
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
       f = self.OpenTraceFile(afile, should_post_file)
       lines = f.readlines()
       line = ''
       i = 0
       revision_num = 0
@@ skipping 128 matching lines @@
 
   class DromaeoPerfTester(DromaeoTester):
     def MoveChromeDriverIfNeeded(self, browser):
       """Move the appropriate version of ChromeDriver onto the path. 
       TODO(efortuna): This is a total hack because the latest version of Chrome
       (Dartium builds) requires a different version of ChromeDriver, that is
       incompatible with the release or beta Chrome and vice versa. Remove these
       shenanigans once we're back to both versions of Chrome using the same
       version of ChromeDriver. IMPORTANT NOTE: This assumes your chromedriver is
       in the default location (inside depot_tools).
+
+      Returns: True if we were successfully able to download a new version of
+      chromedriver and/or move the correct chromedriver into position.
       """
       current_dir = os.getcwd()
       self.test.test_runner.GetArchive('chromedriver')
       path = os.environ['PATH'].split(os.pathsep)
       orig_chromedriver_path = os.path.join(DART_REPO_LOC, 'tools', 'testing',
                                             'orig-chromedriver')
       dartium_chromedriver_path = os.path.join(DART_REPO_LOC, 'tools',
                                                'testing',
                                                'dartium-chromedriver')
       extension = ''
       if platform.system() == 'Windows':
         extension = '.exe'
 
       def MoveChromedriver(depot_tools, copy_to_depot_tools_dir=True,
                             from_path=None):
         if from_path:
           from_dir = from_path + extension
         else:
           from_dir =  os.path.join(orig_chromedriver_path,
                                    'chromedriver' + extension)
         to_dir = os.path.join(depot_tools, 'chromedriver' + extension)
         if not copy_to_depot_tools_dir:
           tmp = to_dir
           to_dir = from_dir
           from_dir = tmp
         print >> sys.stderr, from_dir
         print >> sys.stderr, to_dir
         if not os.path.exists(os.path.dirname(to_dir)):
           os.makedirs(os.path.dirname(to_dir))
+        if not os.path.exists(os.path.dirname(from_dir)):
+          os.makedirs(os.path.dirname(from_dir))
         shutil.copyfile(from_dir, to_dir)
 
       for loc in path:
         if 'depot_tools' in loc:
           if browser == 'chrome':
             if os.path.exists(orig_chromedriver_path):
               MoveChromedriver(loc)
           elif browser == 'dartium':
             if (int(self.test.test_runner.current_revision_num) < 
                 FIRST_CHROMEDRIVER):
               # If we don't have a stashed a different chromedriver just use
               # the regular chromedriver.
+              if not os.path.exists(os.path.dirname(orig_chromedriver_path)):
+                os.makedirs(os.path.dirname(orig_chromedriver_path))
               self.test.test_runner.RunCmd([os.path.join(
                   TOP_LEVEL_DIR, 'tools', 'testing', 'webdriver_test_setup.py'),
                   '-f', '-p', '-s'])
             elif not os.path.exists(dartium_chromedriver_path):
-              stdout, _ = self.test.test_runner.GetArchive('chromedriver')
+              success, _, _ = self.test.test_runner.GetArchive('chromedriver')
+              if not success:
+                return False
             # Move original chromedriver for storage.
             if not os.path.exists(orig_chromedriver_path):
               MoveChromedriver(loc, copy_to_depot_tools_dir=False)
             if self.test.test_runner.current_revision_num >= FIRST_CHROMEDRIVER:
               # Copy Dartium chromedriver into depot_tools
               MoveChromedriver(loc, from_path=os.path.join(
                                 dartium_chromedriver_path, 'chromedriver'))
       os.chdir(current_dir)
+      return True
 
     def RunTests(self):
       """Run dromaeo in the browser."""
       
-      self.test.test_runner.GetArchive('dartium')
+      success, _, _ = self.test.test_runner.GetArchive('dartium')
+      if not success:
+        # Unable to download dartium. Try later.
+        return
 
       # Build tests.
       dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
       current_path = os.getcwd()
       os.chdir(dromaeo_path)
       if os.path.exists('generate_dart2js_tests.py'):
         stdout, _ = self.test.test_runner.RunCmd(
             ['python', 'generate_dart2js_tests.py'])
       else:
         stdout, _ = self.test.test_runner.RunCmd(
             ['python', 'generate_frog_tests.py'])
       os.chdir(current_path)
       if 'Error: Compilation failed' in stdout:
         return
       versions = DromaeoTester.GetDromaeoVersions()
 
       for browser in BrowserTester.GetBrowsers():
-        self.MoveChromeDriverIfNeeded(browser)
+        success = self.MoveChromeDriverIfNeeded(browser)
+        if not success:
+          return
         for version_name in versions:
           if not self.test.IsValidCombination(browser, version_name):
             continue
           version = DromaeoTest.DromaeoPerfTester.GetDromaeoUrlQuery(
               browser, version_name)
           self.test.trace_file = os.path.join(TOP_LEVEL_DIR,
               'tools', 'testing', 'perf_testing', self.test.result_folder_name,
               'dromaeo-%s-%s-%s' % (self.test.cur_time, browser, version_name))
           self.AddSvnRevisionToTrace(self.test.trace_file, browser)
           file_path = '"%s"' % os.path.join(os.getcwd(), dromaeo_path,
@@ skipping 12 matching lines @@
         version = version.replace('frog', 'dart')
       version = version.replace('_','AND')
       tags = DromaeoTester.GetValidDromaeoTags()
       return 'OR'.join([ '%sAND%s' % (version, tag) for tag in tags])
 
 
   class DromaeoFileProcessor(Processor):
     def ProcessFile(self, afile, should_post_file):
       """Comb through the html to find the performance results.
       Returns: True if we successfully posted our data to storage."""
-      print afile
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
 
       bench_dict = self.test.values_dict[browser][version]
 
       f = self.OpenTraceFile(afile, should_post_file)
       lines = f.readlines()
       i = 0
       revision_num = 0
@@ skipping 117 matching lines @@
     if num_results < 10:
       # Run at most two more times.
       if num_results > 8:
         reruns = 10 - num_results
       else:
         reruns = 2
       run = runner.RunTestSequence(revision_num=str(revision_number),
           num_reruns=reruns)
     if num_results >= 10 or run == 0 and num_results + reruns >= 10:
       results_set = UpdateSetOfDoneCls(revision_number)
-    else:
+    elif run != 0:
       return False
     return True
 
   if random.choice([True, False]):
     # Select a random CL number, with greater likelihood of selecting a CL in
     # the more recent history than the distant past (using a simplified weighted
     # bucket algorithm). If that CL has less than 10 runs, run additional. If it
     # already has 10 runs, look for another CL number that is not yet have all
     # of its additional runs (do this up to 15 times).
     tries = 0
@@ skipping 55 matching lines @@
       else:
         if runner.backfill:
           results_set = FillInBackHistory(results_set, runner)
         else:
           time.sleep(200)
   else:
     runner.RunTestSequence()
 
 if __name__ == '__main__':
   main()