Make some changes to the perf tests running script.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import math             
 import optparse
 import os
@@ skipping 76 matching lines @@
       os.chdir(DART_REPO_LOC)
       results, _ = self.run_cmd(['svn', 'st'])
       for line in results.split('\n'):
         if line.startswith('?'):
           to_remove = line.split()[1]
           if os.path.isdir(to_remove):
             shutil.rmtree(to_remove)#, ignore_errors=True)
           else:
             os.remove(to_remove)
 
+  def get_archive(archive_name):
+    """Wrapper around the pulling down a specific archive from Google Storage.
+    Adds a specific revision argument as needed.
+    Returns: The stderr from running this command."""
+    cmd = ['python', os.path.join(DART_REPO_LOC, 'tools', 'get_archive.py'),
+           archive_name]
+    if self.current_revision_num != -1:
+      cmd += ['-r', revision_num]
+    _, stderr = self.test.test_runner.run_cmd(cmd)
+    return stderr
+
   def sync_and_build(self, suites, revision_num=''):
     """Make sure we have the latest version of of the repo, and build it. We
     begin and end standing in DART_REPO_LOC.
 
     Args:
       suites: The set of suites that we wish to build.
 
     Returns:
       err_code = 1 if there was a problem building."""
     os.chdir(dirname(DART_REPO_LOC))
     self.clear_out_unversioned_files()
     if revision_num == '':
       self.run_cmd(['gclient', 'sync'])
     else:
       self.run_cmd(['gclient', 'sync', '-r', revision_num, '-t'])
     
     shutil.copytree(os.path.join(TOP_LEVEL_DIR, 'internal'),
                     os.path.join(DART_REPO_LOC, 'internal'))
     shutil.copy(os.path.join(TOP_LEVEL_DIR, 'tools', 'get_archive.py'),
                     os.path.join(DART_REPO_LOC, 'tools', 'get_archive.py'))
+    shutil.copy(
+        os.path.join(TOP_LEVEL_DIR, 'tools', 'testing', 'run_selenium.py'),
+        os.path.join(DART_REPO_LOC, 'tools', 'testing', 'run_selenium.py'))
 
     if revision_num == '':
       revision_num = search_for_revision(['svn', 'info'])
       if revision_num == -1:
         revision_num = search_for_revision(['git', 'svn', 'info'])
 
-    get_archive_path = os.path.join(DART_REPO_LOC, 'tools', 'get_archive.py')
-    if os.path.exists(get_archive_path):
-      cmd = ['python', get_archive_path, 'sdk']
-      if revision_num != -1:
-        cmd += ['-r', revision_num]
-      _, stderr = self.run_cmd(cmd)
+    self.current_revision_num = revision_num
+    stderr = get_archive('sdk')
     if not os.path.exists(get_archive_path) or 'InvalidUriError' in stderr:
       # Couldn't find the SDK on Google Storage. Build it locally.
 
       # On Windows, the output directory is marked as "Read Only," which causes
       # an error to be thrown when we use shutil.rmtree. This helper function
       # changes the permissions so we can still delete the directory.
       def on_rm_error(func, path, exc_info):
         if os.path.exists(path):
           os.chmod(path, stat.S_IWRITE)
           os.unlink(path)
@@ skipping 21 matching lines @@
     Args:
       dir_name: the directory we will create if it does not exist."""
     dir_path = os.path.join(TOP_LEVEL_DIR, 'tools', 
                             'testing', 'perf_testing', dir_name)
     if not os.path.exists(dir_path):
       os.makedirs(dir_path)
       print 'Creating output directory ', dir_path
 
   def has_new_code(self):
     """Tests if there are any newer versions of files on the server."""
+    if not os.path.exists(DART_REPO_LOC):
+      return True
     os.chdir(DART_REPO_LOC)
     # Pass 'p' in if we have a new certificate for the svn server, we want to
     # (p)ermanently accept it.
     results, _ = self.run_cmd(['svn', 'st', '-u'], std_in='p\r\n')
     for line in results:
       if '*' in line:
         return True
     return False
     
   def get_os_directory(self):
     """Specifies the name of the directory for the testing build of dart, which
     has yet a different naming convention from utils.getBuildRoot(...)."""
     if platform.system() == 'Windows':
       return 'windows'
     elif platform.system() == 'Darwin':
       return 'macos'
     else:
       return 'linux'
 
   def parse_args(self):
     parser = optparse.OptionParser()
     parser.add_option('--suites', '-s', dest='suites', help='Run the specified '
                       'comma-separated test suites from set: %s' % \
                       ','.join(TestBuilder.available_suite_names()), 
                       action='store', default=None)
     parser.add_option('--forever', '-f', dest='continuous', help='Run this scri'
                       'pt forever, always checking for the next svn checkin',
                       action='store_true', default=False)
-    parser.add_option('--incremental', '-i', dest='incremental',
-                      help='Start an an early revision and work your way '
-                      'forward through CLs sequentially', action='store_true',
-                      default=False)
     parser.add_option('--nobuild', '-n', dest='no_build', action='store_true',
                       help='Do not sync with the repository and do not '
                       'rebuild.', default=False)
     parser.add_option('--noupload', '-u', dest='no_upload', action='store_true',
                       help='Do not post the results of the run.', default=False)
     parser.add_option('--notest', '-t', dest='no_test', action='store_true',
                       help='Do not run the tests.', default=False)
     parser.add_option('--verbose', '-v', dest='verbose', help='Print extra '
                       'debug output', action='store_true', default=False)
 
     args, ignored = parser.parse_args()
 
     if not args.suites:
       suites = TestBuilder.available_suite_names()
     else:
       suites = []
       suitelist = args.suites.split(',')
       for name in suitelist:
         if name in TestBuilder.available_suite_names():
           suites.append(name)
         else:
           print ('Error: Invalid suite %s not in ' % name) + \
               '%s' % ','.join(TestBuilder.available_suite_names())
           sys.exit(1)
     self.suite_names = suites
     self.no_build = args.no_build
     self.no_upload = args.no_upload
     self.no_test = args.no_test
     self.verbose = args.verbose
-    return args.continuous, args.incremental
+    return args.continuous
 
   def run_test_sequence(self, revision_num='', num_reruns=1):
     """Run the set of commands to (possibly) build, run, and post the results
     of our tests. Returns 0 on a successful run, 1 if we fail to post results or
     the run failed, -1 if the build is broken.
     """
     suites = []
     success = True
     if not self.no_build and self.sync_and_build(suites, revision_num) == 1:
       return -1 # The build is broken.
@@ skipping 105 matching lines @@
   def __init__(self, test):
     self.test = test
 
   def prepare(self):
     """Perform any initial setup required before the test is run."""
     pass
 
   def add_svn_revision_to_trace(self, outfile, browser = None):
     """Add the svn version number to the provided tracefile."""
     def get_dartium_revision():
-      version_file_name = os.path.join(TOP_LEVEL_DIR, 'client', 'tests',
+      version_file_name = os.path.join(DART_REPO_LOC, 'client', 'tests',
                                        'dartium', 'LAST_VERSION')
       version_file = open(version_file_name, 'r')
       version = version_file.read().split('.')[-2]
       version_file.close()
       return version
 
     if browser and browser == 'dartium':
       revision = get_dartium_revision()
       self.test.test_runner.run_cmd(['echo', 'Revision: ' + revision], outfile)
     else:
@@ skipping 94 matching lines @@
           file_processor)
     self.platform_list = platform_list
     self.platform_type = platform_type
     self.versions = versions
     self.benchmarks = benchmarks 
 
 
 class BrowserTester(Tester):
   @staticmethod
   def get_browsers(add_dartium=True):
-    browsers = ['chrome']#['ff', 'chrome']
+    browsers = ['ff', 'chrome']
     if add_dartium:
-      pass
-      #browsers += ['dartium']
+      browsers += ['dartium']
     has_shell = False
     if platform.system() == 'Darwin':
-      pass
-      #browsers += ['safari']
+      browsers += ['safari']
     if platform.system() == 'Windows':
       browsers += ['ie']
       has_shell = True
-    if 'dartium' in browsers:
-      # Fetch it if necessary.
-      get_dartium = ['python', os.path.join(DART_REPO_LOC, 'tools',
-                     'get_archive.py'), 'dartium']
-      # TODO(vsm): It's inconvenient that run_cmd isn't in scope here.
-      # Perhaps there is a better place to put that or this.
-      subprocess.call(get_dartium, stdout=sys.stdout, stderr=sys.stderr,
-                      shell=has_shell)
     return browsers
 
 
 class CommonBrowserTest(RuntimePerformanceTest):
   """Runs this basic performance tests (Benchpress, some V8 benchmarks) in the
   browser."""
 
   def __init__(self, test_runner):
     """Args:
       test_runner: Reference to the object that notifies us when to run."""
@@ skipping 159 matching lines @@
   def get_dromaeo_benchmarks():
     valid = DromaeoTester.get_valid_dromaeo_tags()
     benchmarks = reduce(lambda l1,l2: l1+l2,
                         [tests for (tag, tests) in
                          DromaeoTester.DROMAEO_BENCHMARKS.values() 
                          if tag in valid])
     return map(DromaeoTester.legalize_filename, benchmarks)
 
   @staticmethod
   def get_dromaeo_versions():
-    # TODO(vsm): why is the js version closing early?
-    return ['dart2js_dom', 'dart2js_html'] 
-    #return ['js', 'dart2js_dom', 'dart2js_html']
+    return ['js', 'dart2js_dom', 'dart2js_html']
 
 
 class DromaeoTest(RuntimePerformanceTest):
   """Runs Dromaeo tests, in the browser."""
   def __init__(self, test_runner):
     super(DromaeoTest, self).__init__(
         self.name(),
-        BrowserTester.get_browsers(),
+        BrowserTester.get_browsers(True),
         'browser',
         DromaeoTester.get_dromaeo_versions(), 
         DromaeoTester.get_dromaeo_benchmarks(), test_runner,
         self.DromaeoPerfTester(self),
         self.DromaeoFileProcessor(self))
 
   @staticmethod
   def name():
     return 'dromaeo'
 
@@ skipping 12 matching lines @@
     def move_chrome_driver_if_needed(self, browser):
       """Move the appropriate version of ChromeDriver onto the path. 
       TODO(efortuna): This is a total hack because the latest version of Chrome
       (Dartium builds) requires a different version of ChromeDriver, that is
       incompatible with the release or beta Chrome and vice versa. Remove these
       shenanigans once we're back to both versions of Chrome using the same
       version of ChromeDriver. IMPORTANT NOTE: This assumes your chromedriver is
       in the default location (inside depot_tools).
       """
       current_dir = os.getcwd()
+      self.test.test_runner.get_archive('chromedriver')
       self.test.test_runner.run_cmd(['python', os.path.join(
-          'tools', 'get_archive.py'), 'chromedriver'])
       path = os.environ['PATH'].split(os.pathsep)
       orig_chromedriver_path = os.path.join(DART_REPO_LOC, 'tools', 'testing',
                                             'orig-chromedriver')
       dartium_chromedriver_path = os.path.join(DART_REPO_LOC, 'tools',
                                                'testing',
                                                'dartium-chromedriver')
       extension = ''
       if platform.system() == 'Windows':
         extension = '.exe'
 
@@ skipping 15 matching lines @@
           os.makedirs(os.path.dirname(to_dir))
         shutil.copyfile(from_dir, to_dir)
 
       for loc in path:
         if 'depot_tools' in loc:
           if browser == 'chrome':
             if os.path.exists(orig_chromedriver_path):
               move_chromedriver(loc)
           elif browser == 'dartium':
             if not os.path.exists(dartium_chromedriver_path):
-              self.test.test_runner.run_cmd(['python',
-                  os.path.join('tools', 'get_archive.py'), 'chromedriver'])
+              self.test.test_runner.get_archive('chromedriver')
             # Move original chromedriver for storage.
             if not os.path.exists(orig_chromedriver_path):
               move_chromedriver(loc, copy_to_depot_tools_dir=False)
             # Copy Dartium chromedriver into depot_tools
             move_chromedriver(loc, from_path=os.path.join(
                               dartium_chromedriver_path, 'chromedriver'))
       os.chdir(current_dir)
 
     def run_tests(self):
       """Run dromaeo in the browser."""
+      
+      self.test.test_runner.get_archive('dartium')
 
       # Build tests.
       dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
       current_path = os.getcwd()
       os.chdir(dromaeo_path)
       self.test.test_runner.run_cmd(['python', 'generate_dart2js_tests.py'])
       os.chdir(current_path)
 
       versions = DromaeoTester.get_dromaeo_versions()
 
@@ skipping 97 matching lines @@
 
 
   class DromaeoSizeTester(DromaeoTester):
     def run_tests(self):
       # Build tests.
       dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')
       current_path = os.getcwd()
       os.chdir(dromaeo_path)
       self.test.test_runner.run_cmd(
           ['python', os.path.join('generate_dart2js_tests.py')])
+      self.test.test_runner.get_archive('dartium')
       os.chdir(current_path)
 
       self.test.trace_file = os.path.join(TOP_LEVEL_DIR,
           'tools', 'testing', 'perf_testing', self.test.result_folder_name,
           self.test.result_folder_name + self.test.cur_time)
       self.add_svn_revision_to_trace(self.test.trace_file)
 
       variants = [
           ('frog_dom', ''),
           ('frog_html', '-html'),
@@ skipping 220 matching lines @@
   for line in output.split('\n'):
     if 'Revision' in line:
       return line.split()[1]
   return -1
 
 def update_set_of_done_cls(revision_num=None):
   """Update the set of CLs that do not need additional performance runs.
   Args:
   revision_num: an additional number to be added to the 'done set'
   """
-  filename = os.path.join(dirname(abspath(__file__)), 'cached_results.txt')
+  filename = os.path.join(TOP_LEVEL_DIR, 'cached_results.txt')
   if not os.path.exists(filename):
     f = open(filename, 'w')
     results = set()
     pickle.dump(results, f)
     f.close()
   f = open(filename, 'r+')
   result_set = pickle.load(f)
   if revision_num:
     f.seek(0)
     result_set.add(revision_num)
     pickle.dump(result_set, f)
   f.close()
   return result_set
 
 def main():
   runner = TestRunner()
-  continuous, incremental = runner.parse_args()
+  continuous = runner.parse_args()
 
   if not os.path.exists(DART_REPO_LOC):
     os.mkdir(dirname(DART_REPO_LOC))
     os.chdir(dirname(DART_REPO_LOC))
     p = subprocess.Popen('gclient config https://dart.googlecode.com/svn/' +
                          'branches/bleeding_edge/deps/all.deps',
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                          shell=True)
     p.communicate()
   if continuous:
@@ skipping 24 matching lines @@
                 a_test.file_processor.get_score_type(benchmark_name))
             if number_of_results < 10 and number_of_results >= 0:
               run = runner.run_test_sequence(revision_num=str(revision_num),
                   num_reruns=(10-number_of_results))
               if run == 0:
                 has_run_extra = True
                 results_set = update_set_of_done_cls(revision_num)
           revision_num -= 1
         # No more extra back-runs to do (for now). Wait for new code.
         time.sleep(200)
-  elif incremental:
-    # This is a temporary measure to backfill old revisions.
-    # TODO(efortuna): Clean this up -- don't hard code numbers, make user
-    # specifiable.
-    revision_num = 9000
-    while revision_num < 10600:
-      run = runner.run_test_sequence(revision_num=str(revision_num),
-          num_reruns=10)
-      revision_num += 1
   else:
     runner.run_test_sequence()
 
 if __name__ == '__main__':
   main()