Add Web Page Replay enabled page cycler tests to pyauto.

chrome/test/functional/perf.py

Index: chrome/test/functional/perf.py
diff --git a/chrome/test/functional/perf.py b/chrome/test/functional/perf.py
index fd299e0ef73bfd9faba408db35ca1c8712315797..55a2b813bf0f206c9430beeb01497835b02b169a 100755
--- a/chrome/test/functional/perf.py
+++ b/chrome/test/functional/perf.py
@@ -48,6 +48,7 @@ import simplejson  # Must be imported after pyauto; located in third_party.
 from netflix import NetflixTestHelper
 import pyauto_utils
 import test_utils
+import webpagereplay
 from youtube import YoutubeTestHelper
 
 
@@ -1733,12 +1734,18 @@ class LiveGamePerfTest(BasePerfTest):
                               'AngryBirds', 'angry_birds')
 
 
-class PageCyclerTest(BasePerfTest):
-  """Tests to run various page cyclers."""
+class BasePageCyclerTest(BasePerfTest):
+  """Page class for page cycler tests.
+
+  Derived classes must implement _StartUrl().

[Nirnimesh, 2012/05/18 19:19:13]

Methods prefixed with _ are considered private. Derived classes cannot be
expected to implement it. Please remove _

Same for some other methods you're overriding.

[slamm_google, 2012/05/18 23:59:28]

Done. Thanks I was unclear on that. I changed _RunPageCyclerTest too.

+  """
 
   # Page Cycler lives in src/data/page_cycler rather than src/chrome/test/data
-  PC_PATH = os.path.join(BasePerfTest.DataDir(), os.pardir, os.pardir,
-                         os.pardir, 'data', 'page_cycler')
+  DATA_PATH = os.path.join(BasePerfTest.DataDir(), os.pardir, os.pardir,
+                           os.pardir, 'data', 'page_cycler')
+  @classmethod

[tonyg, 2012/05/18 15:37:54]

nit: blank line above this

[slamm_google, 2012/05/18 23:59:28]

Done.

+  def DataPath(cls, subdir):
+    return os.path.join(cls.DATA_PATH, subdir)
 
   def ExtraChromeFlags(self):
     """Ensures Chrome is launched with custom flags.
@@ -1750,129 +1757,228 @@ class PageCyclerTest(BasePerfTest):
     # The first two are needed for the test.
     # The plugins argument is to prevent bad scores due to pop-ups from
     # running an old version of something (like Flash).
-    return (super(PageCyclerTest, self).ExtraChromeFlags() +
+    return (super(BasePageCyclerTest, self).ExtraChromeFlags() +
             ['--js-flags="--expose_gc"',
              '--enable-file-cookies',
              '--allow-outdated-plugins'])
 
-  def _PreReadDir(self, dir):
-    """This recursively reads all of the files in a given url directory.
+  def _WaitUntilDone(self, url, iterations):
+    """Check cookies for "__pc_done=1" to know the test is over."""
+    def IsDone():
+      cookies = self.GetCookie(pyauto.GURL(url))  # window 0, tab 0
+      return '__pc_done=1' in cookies
+    self.assertTrue(
+        self.WaitUntil(IsDone, timeout=(60 * iterations), retry_sleep=1),
+        msg='Timed out waiting for page cycler test to complete.')
 
-    The intent is to get them into memory before they are used by the benchmark.
-    """
-    def _PreReadDir(dirname, names):
-      for rfile in names:
-        with open(os.path.join(dirname, rfile)) as fp:
-          fp.read()
+  def _CollectPagesAndTimes(self, url):
+    """Collect the results from the cookies."""
+    pages, times = None, None
+    cookies = self.GetCookie(pyauto.GURL(url))  # window 0, tab 0
+    for cookie in cookies.split(';'):
+      if '__pc_pages' in cookie:
+        pages_str = cookie.split('=', 1)[1]
+        pages = pages_str.split(',')
+      elif '__pc_timings' in cookie:
+        times_str = cookie.split('=', 1)[1]
+        times = [float(t) for t in times_str.split(',')]
+    self.assertTrue(pages and times,
+                    msg='Unable to find test results in cookies: %s' % cookies)
 
-    for root, dirs, files in os.walk(os.path.dirname(dir)):
-      _PreReadDir(root, files)
+  def _IteratePageTimes(self, times, num_pages, iterations):
+    """Regroup the times by the page.
 
-  def setUp(self):
-    self._PreReadDir(os.path.join(self.PC_PATH, 'common'))
-    BasePerfTest.setUp(self)
+    Args:
+      times: e.g. [page1_iter1, page1_iter2, ..., page2_iter1, page2_iter2, ...]
+      num_pages: the number of pages
+      iterations: the number of times for each page
+    Yields:
+      times for one page: [page1_iter1, page1_iter2, ...]
+    """
+    expected_num_times = num_pages * iterations
+    self.assertEqual(
+        expected_num_times, len(times),
+        msg=('len(times) != num_pages * iterations: %s != %s * %s, times=%s' %
+             (len(times), num_pages, iterations, times)))
 
-  def _RunPageCyclerTest(self, dirname, iterations, description):
-    """Runs the specified PageCycler test.
+    next_time = iter(times).next
+    for _ in range(num_pages):
+      yield [next_time() for _ in range(iterations)]
 
-    The final score that is calculated is a geometric mean of the
-    arithmetic means of each site's load time, and we drop the upper
-    20% of the times for each site so they don't skew the mean.
-    The Geometric mean is used for the final score because the time
-    range for any given site may be very different, and we don't want
-    slower sites to weight more heavily than others.
+  def _TrimTimes(self, times, percent):
+    """Return a new list with |percent| number of times trimmed for each page.
 
-    Args:
-      dirname: The directory containing the page cycler test.
-      iterations: How many times to run through the set of pages.
-      description: A string description for the particular test being run.
+    Removes the largest and smallest values.
     """
-    self._PreReadDir(os.path.join(self.PC_PATH, dirname))
+    iterations = len(times)
+    times = sorted(times)
+    logging.debug('Before trimming %d: %s' % num_to_drop, times)
+
+    num_to_trim = int(iterations * float(percent) / 100.0)
+    a = num_to_drop / 2
+    b = iterations - (num_to_trim / 2 + num_to_trim % 2)
+    trimmed_times = times[a:b]
+    logging.debug('After trimming: %s', trimmed_times)
+    return trimmed_times
+
+  def _GetArithmeticMean(self, values):

[tonyg, 2012/05/18 15:37:54]

These math helper functions shouldn't be named as getters and don't could be
made static or better yet moved out of this class. Do we have a more generic
place for things like this?

[slamm_google, 2012/05/18 23:59:28]

I do not see any collection of functions in this directory or higher up.
The closest seems to be ./test_utils.py. I moved the to the top of this file for
now.

BTW, there is an implementation of the geometric mean in
tools/build/scripts/slave/chromium/dom_perf.py.

I also updated this geometric mean calculation to deal with large numbers if
necessary.

+    """Return the arithmetic mean of |values|."""
+    return sum(values) / len(values)
+
+  def _GetGeometricMean(self, values):
+    """Return the geometric mean of |values|."""
+    return reduce(lambda x, y: x * y, values) ** (1.0 / len(values))
+
+  def _ComputeFinalResult(self, times, num_pages, iterations):
+    """The final score that is calculated is a geometric mean of the
+    arithmetic means of each page's load time, and we drop the
+    upper/lower 20% of the times for each page so they don't skew the
+    mean.  The geometric mean is used for the final score because the
+    time range for any given site may be very different, and we don't
+    want slower sites to weight more heavily than others.
+    """
+    page_means = [
+        self._GetArithmeticMean(self._TrimTimes(times, percent=20))
+        for times in self._IteratePageTimes(times, num_pages, iterations)]
+    return self._GetGeometricMean(page_means)
 
-    url = self.GetFileURLForDataPath(os.path.join(self.PC_PATH, dirname),
-                                     'start.html')
+  def _StartUrl(self, test_name, iterations):
+    """Return the URL to used to start the test.
 
-    self.NavigateToURL('%s?auto=1&iterations=%d' % (url, iterations))
+    Subclasses must implement this.
+    """
+    raise NotImplemented
 
-    # Check cookies for "__pc_done=1" to know the test is over.
-    def IsTestDone():
-      cookies = self.GetCookie(pyauto.GURL(url))  # Window 0, tab 0.
-      return '__pc_done=1' in cookies
+  def _RunPageCyclerTest(self, name, description):
+    """Runs the specified PageCycler test.
 

[tonyg, 2012/05/18 15:37:54]

nit: extra line break

[slamm_google, 2012/05/18 23:59:28]

Done.

-    self.assertTrue(
-        self.WaitUntil(IsTestDone, timeout=(60 * iterations), retry_sleep=1),
-        msg='Timed out waiting for page cycler test to complete.')
 
-    # Collect the results from the cookies.
-    site_to_time_list = {}
-    cookies = self.GetCookie(pyauto.GURL(url))  # Window 0, tab 0.
-    site_list = ''
-    time_list = ''
-    for cookie in cookies.split(';'):
-      if '__pc_pages' in cookie:
-        site_list = cookie[cookie.find('=') + 1:]
-      elif '__pc_timings' in cookie:
-        time_list = cookie[cookie.find('=') + 1:]
-    self.assertTrue(site_list and time_list,
-                    msg='Could not find test results in cookies: %s' % cookies)
-    site_list = site_list.split(',')
-    time_list = time_list.split(',')
-    self.assertEqual(iterations, len(time_list) / len(site_list),
-                     msg='Iteration count %d does not match with site/timing '
-                     'lists: %s and %s' % (iterations, site_list, time_list))
-    for site_index, site in enumerate(site_list):
-      site_to_time_list[site] = []
-      for iteration_index in xrange(iterations):
-        site_to_time_list[site].append(
-            float(time_list[iteration_index * len(site_list) + site_index]))
-
-    site_times = []
-    for site, time_list in site_to_time_list.iteritems():
-      sorted_times = sorted(time_list)
-      num_to_drop = int(len(sorted_times) * 0.2)
-      logging.debug('Before dropping %d: ' % num_to_drop)
-      logging.debug(sorted_times)
-      if num_to_drop:
-        sorted_times = sorted_times[:-num_to_drop]
-      logging.debug('After dropping:')
-      logging.debug(sorted_times)
-      # Do an arithmetic mean of the load times for a given page.
-      mean_time = sum(sorted_times) / len(sorted_times)
-      logging.debug('Mean time is: ' + str(mean_time))
-      site_times.append(mean_time)
-
-    logging.info('site times = %s' % site_times)
-    # Compute a geometric mean over the averages for each site.
-    final_result = reduce(lambda x, y: x * y,
-                          site_times) ** (1.0/ len(site_times))
+    Args:
+      name: the page cycler test name (corresponds to a directory or test file)
+      description: a string description for the test
+    """
+    iterations = self._num_iterations
+    start_url = self._StartUrl(name, iterations)
+    self.NavigateToURL(start_url)
+    self._WaitUntilDone(start_url, iterations)
+    pages, times = self._CollectPagesAndTimes(start_url)
+    final_result = self._ComputeFinalResult(times, len(pages), iterations)
     logging.info('%s page cycler final result: %f' %
                  (description, final_result))
     self._OutputPerfGraphValue(description + '_PageCycler', final_result,
                                'milliseconds', graph_name='PageCycler')
 
+
+class PageCyclerTest(BasePageCyclerTest):
+  """Tests to run various page cyclers."""
+
+  def _PreReadDataDir(self, subdir):
+    """This recursively reads all of the files in a given url directory.
+
+    The intent is to get them into memory before they are used by the benchmark.
+
+    Args:
+      subdir: a subdirectory of the page cycler data directory.
+    """
+    def _PreReadDir(dirname, names):
+      for rfile in names:
+        with open(os.path.join(dirname, rfile)) as fp:
+          fp.read()
+    for root, dirs, files in os.walk(self.DataPath(subdir)):
+      _PreReadDir(root, files)
+
+  def _StartUrl(self, test_name, iterations):
+    return self.GetFileURLForDataPath(
+        self.DataPath(test_name),
+        'start.html?auto=1&iterations=%d' % iterations)
+
+  def _RunPageCyclerTest(self, dirname, description):
+    """Runs the specified PageCycler test.
+
+    Args:
+      dirname: directory containing the page cycler test
+      description: a string description for the test
+    """
+    self._PreReadDataDir('common')
+    self._PreReadDataDir(dirname)
+    super(PageCyclerTest, self)._RunPageCyclerTest(dirname, description)
+
   def testMoreJSFile(self):
-    self._RunPageCyclerTest('morejs', self._num_iterations, 'MoreJSFile')
+    self._RunPageCyclerTest('morejs', 'MoreJSFile')
 
   def testAlexaFile(self):
-    self._RunPageCyclerTest('alexa_us', self._num_iterations, 'Alexa_usFile')
+    self._RunPageCyclerTest('alexa_us', 'Alexa_usFile')
 
   def testBloatFile(self):
-    self._RunPageCyclerTest('bloat', self._num_iterations, 'BloatFile')
+    self._RunPageCyclerTest('bloat', 'BloatFile')
 
   def testDHTMLFile(self):
-    self._RunPageCyclerTest('dhtml', self._num_iterations, 'DhtmlFile')
+    self._RunPageCyclerTest('dhtml', 'DhtmlFile')
 
   def testIntl1File(self):
-    self._RunPageCyclerTest('intl1', self._num_iterations, 'Intl1File')
+    self._RunPageCyclerTest('intl1', 'Intl1File')
 
   def testIntl2File(self):
-    self._RunPageCyclerTest('intl2', self._num_iterations, 'Intl2File')
+    self._RunPageCyclerTest('intl2', 'Intl2File')
 
   def testMozFile(self):
-    self._RunPageCyclerTest('moz', self._num_iterations, 'MozFile')
+    self._RunPageCyclerTest('moz', 'MozFile')
 
   def testMoz2File(self):
-    self._RunPageCyclerTest('moz2', self._num_iterations, 'Moz2File')
+    self._RunPageCyclerTest('moz2', 'Moz2File')
+
+
+class WebPageReplayPageCyclerTest(BasePageCyclerTest):

[tonyg, 2012/05/18 15:37:54]

Please add a class level docstring.

[slamm_google, 2012/05/18 23:59:28]

Done.

+  _TEST_EXE_NAME = 'perf_py'
+  _IS_DNS_FORWARDED = False
+
+  def ChromiumPaths(self, test_name=None):
+    return webpagereplay.ChromiumPaths(
+        TEST_EXE_NAME=self._TEST_EXE_NAME, TEST_NAME=test_name)
+
+  def ExtraChromeFlags(self):
+    """Ensures Chrome is launched with custom flags.
+
+    Returns:
+      A list of extra flags to pass to Chrome when it is launched.
+    """
+    flags = super(WebPageReplayPageCyclerTest, self).ExtraChromeFlags()
+    chromium_paths = self.ChromiumPaths()
+    webpagereplay.ChromeFlags(
+        flags,
+        extension_path=chromium_paths['extension'],
+        is_dns_forwarded=self._IS_DNS_FORWARDED)
+    return flags
+
+  def _StartUrl(self, test_name, iterations):
+    chromium_paths = self.ChromiumPaths(test_name=test_name)
+    return chromium_paths.GetStartUrl(iterations, use_auto=True)
+
+  def _RunPageCyclerTest(self, name, description):
+    """Runs the specified PageCycler test.
+
+    Args:
+      name: name for archive and config files: |name|.wpr and |name|.js.
+      description: a string description for the test
+    """
+    chromium_paths = self.ChromiumPaths(test_name=name)
+    replay_options = []
+    if not self._IS_DNS_FORWARDED:
+      replay_options.append('--no-dns_forwarding')
+    is_record_mode = False  # TODO(slamm): get from environment variable?
+    if is_record_mode:
+      replay_options.append('--record')
+    with webpagereplay.ReplayServer(
+        chromium_paths['replay'],
+        chromium_paths.GetArchivePath(),
+        chromium_paths['logs'],
+        replay_options):
+      super_self = super(WebPageReplayPageCyclerTest, self)
+      super_self._RunPageCyclerTest(name, description)
+

[tonyg, 2012/05/18 15:37:54]

nit: extra line break

[slamm_google, 2012/05/18 23:59:28]

Done.

+
+  def testWpr2012Q2(self):
+    self._RunPageCyclerTest('2012Q2', 'Wpr2012Q2')

[dennis_jeffrey, 2012/05/18 00:28:41]

Is this expected to be able to run on ChromeOS?

By default, any tests added to this file will start running on ChromeOS once the
changes get checked in, unless they're explicitly disabled on ChromeOS in the
PYAUTO_TESTS file (in the same directory as the current file).  If you need to
disable this test there, look for the suite called "PERFORMANCE" in that file. 
For chromeOS, it includes everything in perf.py except for perf.GPUPerfTest.

[slamm_google, 2012/05/18 23:59:29]

Thanks for the heads-up. 

Does this python code run directly on ChromeOS, or is it run on a separate test
machine? In the former case, the code will likely work. In the latter case, the
code would need to know the test host (name or IP).

Web Page Replay starts proxies for http and https. Chrome is pointed to the
proxy with the following options:

    --host-resolver-rules=MAP * %s' % REPLAY_HOST
    --testing-fixed-http-port=%s' % HTTP_PORT
    --testing-fixed-https-port=%s' % HTTPS_PORT

In the Chrome case, REPLAY_HOST is 127.0.0.1.

BTW, I would actually prefer to have an option like --testing-fixed-dns-port so
Chrome/ChromeOS would do DNS lookups against Web Page Replay.

[Sonny, 2012/05/19 00:50:10]

It runs directly on ChromeOS, and as Dennis mentioned it would start getting run
nightly -- would that work with whatever defaults we have?

 
 
 class MemoryTest(BasePerfTest):