[chromedriver] Add some simple perf benchmarks.

chrome/test/chromedriver/run_py_tests.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """End to end tests for ChromeDriver."""
 
 import base64
 import optparse
 import os
@@ skipping 78 matching lines @@
         'ChromeDriverTest.testShouldHandleNewWindowLoadingProperly',
         # https://code.google.com/p/chromedriver/issues/detail?id=259
         'ChromeDriverTest.testSendKeysToElement',
         # https://code.google.com/p/chromedriver/issues/detail?id=270
         'ChromeDriverTest.testPopups',
         # https://code.google.com/p/chromedriver/issues/detail?id=298
         'ChromeDriverTest.testWindowPosition',
         'ChromeDriverTest.testWindowSize',
         'ChromeDriverTest.testWindowMaximize',
         'ChromeLogPathCapabilityTest.testChromeLogPath',
+        # Don't enable perf testing on Android yet.
+        'PerfTest.testSessionStartTime',
+        'PerfTest.testSessionStopTime',
+        'PerfTest.testColdExecuteScript',
     ]
 )
 _ANDROID_NEGATIVE_FILTER['org.chromium.chrome.testshell'] = (
     _ANDROID_NEGATIVE_FILTER['com.google.android.apps.chrome'] + []
 )
 
 
 class ChromeDriverBaseTest(unittest.TestCase):
   """Base class for testing chromedriver functionalities."""
 
   def __init__(self, *args, **kwargs):
     super(ChromeDriverBaseTest, self).__init__(*args, **kwargs)
     self._drivers = []
 
   def tearDown(self):
     for driver in self._drivers:
       try:
         driver.Quit()
-      except chromedriver.ChromeDriverException:
+      except:
         pass
 
-  def CreateDriver(self, **kwargs):
-    driver = chromedriver.ChromeDriver(_CHROMEDRIVER_SERVER_URL,
+  def CreateDriver(self, server_url=None, **kwargs):
+    if server_url is None:
+      server_url = _CHROMEDRIVER_SERVER_URL
+    driver = chromedriver.ChromeDriver(server_url,
                                        chrome_binary=_CHROME_BINARY,
                                        android_package=_ANDROID_PACKAGE,
                                        **kwargs)
     self._drivers += [driver]
     return driver
 
 
 class ChromeDriverTest(ChromeDriverBaseTest):
   """End to end tests for ChromeDriver."""
 
@@ skipping 469 matching lines @@
 
     Verifies that a log message is written into the specified log file.
     """
     tmp_log_path = tempfile.NamedTemporaryFile()
     driver = self.CreateDriver(chrome_log_path=tmp_log_path.name)
     driver.ExecuteScript('console.info("%s")' % self.LOG_MESSAGE)
     driver.Quit()
     self.assertTrue(self.LOG_MESSAGE in open(tmp_log_path.name).read())
 
 
+class PerfTest(ChromeDriverBaseTest):
+  """Tests for ChromeDriver perf."""
+  def setUp(self):
+    self.assertTrue(_REFERENCE_CHROMEDRIVER is not None,
+                    'must supply a reference-chromedriver arg')
+
+  def _RunDriverPerfTest(self, name, test_func):
+    """Runs a perf test comparing a reference and new ChromeDriver server.
+
+    Args:
+      name: The name of the perf test.
+      test_func: Called with the server url to perform the test action. Must
+                 return the time elapsed.
+    """
+    class Results(object):
+      ref = []
+      new = []
+
+    ref_server = chromedriver.Server(_REFERENCE_CHROMEDRIVER)
+    results = Results()
+    result_url_pairs = zip([results.new, results.ref],
+                           [_CHROMEDRIVER_SERVER_URL, ref_server.GetUrl()])
+    for iteration in range(30):
+      for result, url in result_url_pairs:
+        result += [test_func(url)]
+      # Reverse the order for the next run.
+      result_url_pairs = result_url_pairs[::-1]
+
+    def PrintResult(build, result):
+      mean = sum(result) / len(result)
+      avg_dev = sum([abs(sample - mean) for sample in result]) / len(result)
+      print 'perf result', build, name, mean, avg_dev, result
+      util.AddBuildStepText(u'%s %s: %.3f\u00b1%.3f' % (
+          build, name, mean, avg_dev))
+
+    # Discard first result, which may be off due to cold start.
+    PrintResult('new', results.new[1:])
+    PrintResult('ref', results.ref[1:])
+
+  def testSessionStartTime(self):
+    def Run(url):
+      start = time.time()
+      driver = self.CreateDriver(url)
+      end = time.time()
+      driver.Quit()
+      return end - start
+    self._RunDriverPerfTest('session start', Run)
+
+  def testSessionStopTime(self):
+    def Run(url):
+      driver = self.CreateDriver(url)
+      start = time.time()
+      driver.Quit()
+      end = time.time()
+      return end - start
+    self._RunDriverPerfTest('session stop', Run)
+
+  def testColdExecuteScript(self):
+    def Run(url):
+      driver = self.CreateDriver(url)
+      start = time.time()
+      driver.ExecuteScript('return 1')
+      end = time.time()
+      driver.Quit()
+      return end - start
+    self._RunDriverPerfTest('cold exe js', Run)
+
 if __name__ == '__main__':
   parser = optparse.OptionParser()
   parser.add_option(
       '', '--chromedriver',
       help='Path to chromedriver server (REQUIRED!)')
   parser.add_option(
+      '', '--reference-chromedriver',
+      help='Path to the reference chromedriver server')
+  parser.add_option(
       '', '--chrome', help='Path to a build of the chrome binary')
   parser.add_option(
       '', '--chrome-version', default='HEAD',
       help='Version of chrome. Default is \'HEAD\'.')
   parser.add_option(
       '', '--filter', type='string', default='*',
       help=('Filter for specifying what tests to run, "*" will run all. E.g., '
             '*testStartStop'))
   parser.add_option(
       '', '--android-package', help='Android package name')
   options, args = parser.parse_args()
 
   if not options.chromedriver or not os.path.exists(options.chromedriver):
     parser.error('chromedriver is required or the given path is invalid.' +
                  'Please run "%s --help" for help' % __file__)
 
   server = chromedriver.Server(os.path.abspath(options.chromedriver))
   global _CHROMEDRIVER_SERVER_URL
   _CHROMEDRIVER_SERVER_URL = server.GetUrl()
 
+  global _REFERENCE_CHROMEDRIVER
+  _REFERENCE_CHROMEDRIVER = options.reference_chromedriver
+
   global _CHROME_BINARY
   if options.chrome:
     _CHROME_BINARY = os.path.abspath(options.chrome)
   else:
     _CHROME_BINARY = None
 
   global _ANDROID_PACKAGE
   _ANDROID_PACKAGE = options.android_package
 
   if options.filter == '*':
     if _ANDROID_PACKAGE:
       negative_filter = _ANDROID_NEGATIVE_FILTER[_ANDROID_PACKAGE]
     else:
       negative_filter = _DESKTOP_NEGATIVE_FILTER[options.chrome_version]
     options.filter = '*-' + ':__main__.'.join([''] + negative_filter)
 
   all_tests_suite = unittest.defaultTestLoader.loadTestsFromModule(
       sys.modules[__name__])
   tests = unittest_util.FilterTestSuite(all_tests_suite, options.filter)
   ChromeDriverTest.GlobalSetUp()
   result = unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(tests)
   ChromeDriverTest.GlobalTearDown()
   sys.exit(len(result.failures) + len(result.errors))