Add Web Page Replay enabled page cycler tests to pyauto.

chrome/test/functional/perf.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Basic pyauto performance tests.
 
 For tests that need to be run for multiple iterations (e.g., so that average
 and standard deviation values can be reported), the default number of iterations
 run for each of these tests is specified by |_DEFAULT_NUM_ITERATIONS|.
@@ skipping 30 matching lines @@
 import urllib2
 import urlparse
 
 import pyauto_functional  # Must be imported before pyauto.
 import pyauto
 import simplejson  # Must be imported after pyauto; located in third_party.
 
 from netflix import NetflixTestHelper
 import pyauto_utils
 import test_utils
+import webpagereplay
 from youtube import YoutubeTestHelper
 
 
 class BasePerfTest(pyauto.PyUITest):
   """Base class for performance tests."""
 
   _DEFAULT_NUM_ITERATIONS = 10  # Keep synced with desktopui_PyAutoPerfTests.py.
   _DEFAULT_MAX_TIMEOUT_COUNT = 10
   _PERF_OUTPUT_MARKER_PRE = '_PERF_PRE_'
   _PERF_OUTPUT_MARKER_POST = '_PERF_POST_'
@@ skipping 1665 matching lines @@
     logging.info('Total v8 heap size: %f MB', v8_heap_size)
     self._OutputPerfGraphValue(description + 'V8HeapSize', v8_heap_size, 'MB',
                                graph_name + '_v8_heap_size')
 
   def testAngryBirds(self):
     """Measures performance for Angry Birds."""
     self._RunLiveGamePerfTest('http://chrome.angrybirds.com', 'Angry Birds',
                               'AngryBirds', 'angry_birds')
 
 
-class PageCyclerTest(BasePerfTest):
-  """Tests to run various page cyclers."""
+class BasePageCyclerTest(BasePerfTest):
+  """Page class for page cycler tests.
+
+  Derived classes must implement _StartUrl().

[Nirnimesh, 2012/05/18 19:19:13]

Methods prefixed with _ are considered private. Derived classes cannot be
expected to implement it. Please remove _

Same for some other methods you're overriding.

[slamm_google, 2012/05/18 23:59:28]

Done. Thanks I was unclear on that. I changed _RunPageCyclerTest too.

+  """
 
   # Page Cycler lives in src/data/page_cycler rather than src/chrome/test/data
-  PC_PATH = os.path.join(BasePerfTest.DataDir(), os.pardir, os.pardir,
-                         os.pardir, 'data', 'page_cycler')
+  DATA_PATH = os.path.join(BasePerfTest.DataDir(), os.pardir, os.pardir,
+                           os.pardir, 'data', 'page_cycler')
+  @classmethod

[tonyg, 2012/05/18 15:37:54]

nit: blank line above this

[slamm_google, 2012/05/18 23:59:28]

Done.

+  def DataPath(cls, subdir):
+    return os.path.join(cls.DATA_PATH, subdir)
 
   def ExtraChromeFlags(self):
     """Ensures Chrome is launched with custom flags.
 
     Returns:
       A list of extra flags to pass to Chrome when it is launched.
     """
     # Extra flags required to run these tests.
     # The first two are needed for the test.
     # The plugins argument is to prevent bad scores due to pop-ups from
     # running an old version of something (like Flash).
-    return (super(PageCyclerTest, self).ExtraChromeFlags() +
+    return (super(BasePageCyclerTest, self).ExtraChromeFlags() +
             ['--js-flags="--expose_gc"',
              '--enable-file-cookies',
              '--allow-outdated-plugins'])
 
-  def _PreReadDir(self, dir):
+  def _WaitUntilDone(self, url, iterations):
+    """Check cookies for "__pc_done=1" to know the test is over."""
+    def IsDone():
+      cookies = self.GetCookie(pyauto.GURL(url))  # window 0, tab 0
+      return '__pc_done=1' in cookies
+    self.assertTrue(
+        self.WaitUntil(IsDone, timeout=(60 * iterations), retry_sleep=1),
+        msg='Timed out waiting for page cycler test to complete.')
+
+  def _CollectPagesAndTimes(self, url):
+    """Collect the results from the cookies."""
+    pages, times = None, None
+    cookies = self.GetCookie(pyauto.GURL(url))  # window 0, tab 0
+    for cookie in cookies.split(';'):
+      if '__pc_pages' in cookie:
+        pages_str = cookie.split('=', 1)[1]
+        pages = pages_str.split(',')
+      elif '__pc_timings' in cookie:
+        times_str = cookie.split('=', 1)[1]
+        times = [float(t) for t in times_str.split(',')]
+    self.assertTrue(pages and times,
+                    msg='Unable to find test results in cookies: %s' % cookies)
+
+  def _IteratePageTimes(self, times, num_pages, iterations):
+    """Regroup the times by the page.
+
+    Args:
+      times: e.g. [page1_iter1, page1_iter2, ..., page2_iter1, page2_iter2, ...]
+      num_pages: the number of pages
+      iterations: the number of times for each page
+    Yields:
+      times for one page: [page1_iter1, page1_iter2, ...]
+    """
+    expected_num_times = num_pages * iterations
+    self.assertEqual(
+        expected_num_times, len(times),
+        msg=('len(times) != num_pages * iterations: %s != %s * %s, times=%s' %
+             (len(times), num_pages, iterations, times)))
+
+    next_time = iter(times).next
+    for _ in range(num_pages):
+      yield [next_time() for _ in range(iterations)]
+
+  def _TrimTimes(self, times, percent):
+    """Return a new list with |percent| number of times trimmed for each page.
+
+    Removes the largest and smallest values.
+    """
+    iterations = len(times)
+    times = sorted(times)
+    logging.debug('Before trimming %d: %s' % num_to_drop, times)
+
+    num_to_trim = int(iterations * float(percent) / 100.0)
+    a = num_to_drop / 2
+    b = iterations - (num_to_trim / 2 + num_to_trim % 2)
+    trimmed_times = times[a:b]
+    logging.debug('After trimming: %s', trimmed_times)
+    return trimmed_times
+
+  def _GetArithmeticMean(self, values):

[tonyg, 2012/05/18 15:37:54]

These math helper functions shouldn't be named as getters and don't could be
made static or better yet moved out of this class. Do we have a more generic
place for things like this?

[slamm_google, 2012/05/18 23:59:28]

I do not see any collection of functions in this directory or higher up.
The closest seems to be ./test_utils.py. I moved the to the top of this file for
now.

BTW, there is an implementation of the geometric mean in
tools/build/scripts/slave/chromium/dom_perf.py.

I also updated this geometric mean calculation to deal with large numbers if
necessary.

+    """Return the arithmetic mean of |values|."""
+    return sum(values) / len(values)
+
+  def _GetGeometricMean(self, values):
+    """Return the geometric mean of |values|."""
+    return reduce(lambda x, y: x * y, values) ** (1.0 / len(values))
+
+  def _ComputeFinalResult(self, times, num_pages, iterations):
+    """The final score that is calculated is a geometric mean of the
+    arithmetic means of each page's load time, and we drop the
+    upper/lower 20% of the times for each page so they don't skew the
+    mean.  The geometric mean is used for the final score because the
+    time range for any given site may be very different, and we don't
+    want slower sites to weight more heavily than others.
+    """
+    page_means = [
+        self._GetArithmeticMean(self._TrimTimes(times, percent=20))
+        for times in self._IteratePageTimes(times, num_pages, iterations)]
+    return self._GetGeometricMean(page_means)
+
+  def _StartUrl(self, test_name, iterations):
+    """Return the URL to used to start the test.
+
+    Subclasses must implement this.
+    """
+    raise NotImplemented
+
+  def _RunPageCyclerTest(self, name, description):
+    """Runs the specified PageCycler test.
+

[tonyg, 2012/05/18 15:37:54]

nit: extra line break

[slamm_google, 2012/05/18 23:59:28]

Done.

+
+    Args:
+      name: the page cycler test name (corresponds to a directory or test file)
+      description: a string description for the test
+    """
+    iterations = self._num_iterations
+    start_url = self._StartUrl(name, iterations)
+    self.NavigateToURL(start_url)
+    self._WaitUntilDone(start_url, iterations)
+    pages, times = self._CollectPagesAndTimes(start_url)
+    final_result = self._ComputeFinalResult(times, len(pages), iterations)
+    logging.info('%s page cycler final result: %f' %
+                 (description, final_result))
+    self._OutputPerfGraphValue(description + '_PageCycler', final_result,
+                               'milliseconds', graph_name='PageCycler')
+
+
+class PageCyclerTest(BasePageCyclerTest):
+  """Tests to run various page cyclers."""
+
+  def _PreReadDataDir(self, subdir):
     """This recursively reads all of the files in a given url directory.
 
     The intent is to get them into memory before they are used by the benchmark.
+
+    Args:
+      subdir: a subdirectory of the page cycler data directory.
     """
     def _PreReadDir(dirname, names):
       for rfile in names:
         with open(os.path.join(dirname, rfile)) as fp:
           fp.read()
-
-    for root, dirs, files in os.walk(os.path.dirname(dir)):
+    for root, dirs, files in os.walk(self.DataPath(subdir)):
       _PreReadDir(root, files)
 
-  def setUp(self):
-    self._PreReadDir(os.path.join(self.PC_PATH, 'common'))
-    BasePerfTest.setUp(self)
-
-  def _RunPageCyclerTest(self, dirname, iterations, description):
+  def _StartUrl(self, test_name, iterations):
+    return self.GetFileURLForDataPath(
+        self.DataPath(test_name),
+        'start.html?auto=1&iterations=%d' % iterations)
+
+  def _RunPageCyclerTest(self, dirname, description):
     """Runs the specified PageCycler test.
 
-    The final score that is calculated is a geometric mean of the
-    arithmetic means of each site's load time, and we drop the upper
-    20% of the times for each site so they don't skew the mean.
-    The Geometric mean is used for the final score because the time
-    range for any given site may be very different, and we don't want
-    slower sites to weight more heavily than others.
-
-    Args:
-      dirname: The directory containing the page cycler test.
-      iterations: How many times to run through the set of pages.
-      description: A string description for the particular test being run.
-    """
-    self._PreReadDir(os.path.join(self.PC_PATH, dirname))
-
-    url = self.GetFileURLForDataPath(os.path.join(self.PC_PATH, dirname),
-                                     'start.html')
-
-    self.NavigateToURL('%s?auto=1&iterations=%d' % (url, iterations))
-
-    # Check cookies for "__pc_done=1" to know the test is over.
-    def IsTestDone():
-      cookies = self.GetCookie(pyauto.GURL(url))  # Window 0, tab 0.
-      return '__pc_done=1' in cookies
-
-    self.assertTrue(
-        self.WaitUntil(IsTestDone, timeout=(60 * iterations), retry_sleep=1),
-        msg='Timed out waiting for page cycler test to complete.')
-
-    # Collect the results from the cookies.
-    site_to_time_list = {}
-    cookies = self.GetCookie(pyauto.GURL(url))  # Window 0, tab 0.
-    site_list = ''
-    time_list = ''
-    for cookie in cookies.split(';'):
-      if '__pc_pages' in cookie:
-        site_list = cookie[cookie.find('=') + 1:]
-      elif '__pc_timings' in cookie:
-        time_list = cookie[cookie.find('=') + 1:]
-    self.assertTrue(site_list and time_list,
-                    msg='Could not find test results in cookies: %s' % cookies)
-    site_list = site_list.split(',')
-    time_list = time_list.split(',')
-    self.assertEqual(iterations, len(time_list) / len(site_list),
-                     msg='Iteration count %d does not match with site/timing '
-                     'lists: %s and %s' % (iterations, site_list, time_list))
-    for site_index, site in enumerate(site_list):
-      site_to_time_list[site] = []
-      for iteration_index in xrange(iterations):
-        site_to_time_list[site].append(
-            float(time_list[iteration_index * len(site_list) + site_index]))
-
-    site_times = []
-    for site, time_list in site_to_time_list.iteritems():
-      sorted_times = sorted(time_list)
-      num_to_drop = int(len(sorted_times) * 0.2)
-      logging.debug('Before dropping %d: ' % num_to_drop)
-      logging.debug(sorted_times)
-      if num_to_drop:
-        sorted_times = sorted_times[:-num_to_drop]
-      logging.debug('After dropping:')
-      logging.debug(sorted_times)
-      # Do an arithmetic mean of the load times for a given page.
-      mean_time = sum(sorted_times) / len(sorted_times)
-      logging.debug('Mean time is: ' + str(mean_time))
-      site_times.append(mean_time)
-
-    logging.info('site times = %s' % site_times)
-    # Compute a geometric mean over the averages for each site.
-    final_result = reduce(lambda x, y: x * y,
-                          site_times) ** (1.0/ len(site_times))
-    logging.info('%s page cycler final result: %f' %
-                 (description, final_result))
-    self._OutputPerfGraphValue(description + '_PageCycler', final_result,
-                               'milliseconds', graph_name='PageCycler')
+    Args:
+      dirname: directory containing the page cycler test
+      description: a string description for the test
+    """
+    self._PreReadDataDir('common')
+    self._PreReadDataDir(dirname)
+    super(PageCyclerTest, self)._RunPageCyclerTest(dirname, description)
 
   def testMoreJSFile(self):
-    self._RunPageCyclerTest('morejs', self._num_iterations, 'MoreJSFile')
+    self._RunPageCyclerTest('morejs', 'MoreJSFile')
 
   def testAlexaFile(self):
-    self._RunPageCyclerTest('alexa_us', self._num_iterations, 'Alexa_usFile')
+    self._RunPageCyclerTest('alexa_us', 'Alexa_usFile')
 
   def testBloatFile(self):
-    self._RunPageCyclerTest('bloat', self._num_iterations, 'BloatFile')
+    self._RunPageCyclerTest('bloat', 'BloatFile')
 
   def testDHTMLFile(self):
-    self._RunPageCyclerTest('dhtml', self._num_iterations, 'DhtmlFile')
+    self._RunPageCyclerTest('dhtml', 'DhtmlFile')
 
   def testIntl1File(self):
-    self._RunPageCyclerTest('intl1', self._num_iterations, 'Intl1File')
+    self._RunPageCyclerTest('intl1', 'Intl1File')
 
   def testIntl2File(self):
-    self._RunPageCyclerTest('intl2', self._num_iterations, 'Intl2File')
+    self._RunPageCyclerTest('intl2', 'Intl2File')
 
   def testMozFile(self):
-    self._RunPageCyclerTest('moz', self._num_iterations, 'MozFile')
+    self._RunPageCyclerTest('moz', 'MozFile')
 
   def testMoz2File(self):
-    self._RunPageCyclerTest('moz2', self._num_iterations, 'Moz2File')
+    self._RunPageCyclerTest('moz2', 'Moz2File')
+
+
+class WebPageReplayPageCyclerTest(BasePageCyclerTest):

[tonyg, 2012/05/18 15:37:54]

Please add a class level docstring.

[slamm_google, 2012/05/18 23:59:28]

Done.

+  _TEST_EXE_NAME = 'perf_py'
+  _IS_DNS_FORWARDED = False
+
+  def ChromiumPaths(self, test_name=None):
+    return webpagereplay.ChromiumPaths(
+        TEST_EXE_NAME=self._TEST_EXE_NAME, TEST_NAME=test_name)
+
+  def ExtraChromeFlags(self):
+    """Ensures Chrome is launched with custom flags.
+
+    Returns:
+      A list of extra flags to pass to Chrome when it is launched.
+    """
+    flags = super(WebPageReplayPageCyclerTest, self).ExtraChromeFlags()
+    chromium_paths = self.ChromiumPaths()
+    webpagereplay.ChromeFlags(
+        flags,
+        extension_path=chromium_paths['extension'],
+        is_dns_forwarded=self._IS_DNS_FORWARDED)
+    return flags
+
+  def _StartUrl(self, test_name, iterations):
+    chromium_paths = self.ChromiumPaths(test_name=test_name)
+    return chromium_paths.GetStartUrl(iterations, use_auto=True)
+
+  def _RunPageCyclerTest(self, name, description):
+    """Runs the specified PageCycler test.
+
+    Args:
+      name: name for archive and config files: |name|.wpr and |name|.js.
+      description: a string description for the test
+    """
+    chromium_paths = self.ChromiumPaths(test_name=name)
+    replay_options = []
+    if not self._IS_DNS_FORWARDED:
+      replay_options.append('--no-dns_forwarding')
+    is_record_mode = False  # TODO(slamm): get from environment variable?
+    if is_record_mode:
+      replay_options.append('--record')
+    with webpagereplay.ReplayServer(
+        chromium_paths['replay'],
+        chromium_paths.GetArchivePath(),
+        chromium_paths['logs'],
+        replay_options):
+      super_self = super(WebPageReplayPageCyclerTest, self)
+      super_self._RunPageCyclerTest(name, description)
+

[tonyg, 2012/05/18 15:37:54]

nit: extra line break

[slamm_google, 2012/05/18 23:59:28]

Done.

+
+  def testWpr2012Q2(self):
+    self._RunPageCyclerTest('2012Q2', 'Wpr2012Q2')

[dennis_jeffrey, 2012/05/18 00:28:41]

Is this expected to be able to run on ChromeOS?

By default, any tests added to this file will start running on ChromeOS once the
changes get checked in, unless they're explicitly disabled on ChromeOS in the
PYAUTO_TESTS file (in the same directory as the current file).  If you need to
disable this test there, look for the suite called "PERFORMANCE" in that file. 
For chromeOS, it includes everything in perf.py except for perf.GPUPerfTest.

[slamm_google, 2012/05/18 23:59:29]

Thanks for the heads-up. 

Does this python code run directly on ChromeOS, or is it run on a separate test
machine? In the former case, the code will likely work. In the latter case, the
code would need to know the test host (name or IP).

Web Page Replay starts proxies for http and https. Chrome is pointed to the
proxy with the following options:

    --host-resolver-rules=MAP * %s' % REPLAY_HOST
    --testing-fixed-http-port=%s' % HTTP_PORT
    --testing-fixed-https-port=%s' % HTTPS_PORT

In the Chrome case, REPLAY_HOST is 127.0.0.1.

BTW, I would actually prefer to have an option like --testing-fixed-dns-port so
Chrome/ChromeOS would do DNS lookups against Web Page Replay.

[Sonny, 2012/05/19 00:50:10]

It runs directly on ChromeOS, and as Dennis mentioned it would start getting run
nightly -- would that work with whatever defaults we have?

 
 
 class MemoryTest(BasePerfTest):
   """Tests to measure memory consumption under different usage scenarios."""
 
   def ExtraChromeFlags(self):
     """Launches Chrome with custom flags.
 
     Returns:
       A list of extra flags to pass to Chrome when it is launched.
@@ skipping 322 matching lines @@
     """Identifies the port number to which the server is currently bound.
 
     Returns:
       The numeric port number to which the server is currently bound.
     """
     return self._server.server_address[1]
 
 
 if __name__ == '__main__':
   pyauto_functional.Main()